def do_download_for_worker(book, options, merge, notification=lambda x, y: x):
'''
Child job, to download story when run as a worker job
'''
from calibre_plugins.fanficfare_plugin import FanFicFareBase
fffbase = FanFicFareBase(options['plugin_path'])
with fffbase: # so the sys.path was modified while loading the
# plug impl.
from calibre_plugins.fanficfare_plugin.dialogs import (
NotGoingToDownload, OVERWRITE, OVERWRITEALWAYS, UPDATE,
UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
from calibre_plugins.fanficfare_plugin.fff_util import (
get_fff_adapter, get_fff_config)
try:
## No need to download at all. Can happen now due to
## collision moving into book for CALIBREONLY changing to
## ADDNEW when story URL not in library.
if book['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
logger.info("Skipping CALIBREONLY 'update' down inside worker")
return book
book['comment'] = _('Download started...')
configuration = get_fff_config(book['url'], options['fileform'],
options['personal.ini'])
if configuration.getConfig('use_ssl_unverified_context'):
## monkey patch to avoid SSL bug. dupliated from
## fff_plugin.py because bg jobs run in own process
## space.
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
if not options[
'updateepubcover'] and 'epub_for_update' in book and book[
'collision'] in (UPDATE, UPDATEALWAYS):
configuration.set("overrides", "never_make_cover", "true")
# images only for epub, html, even if the user mistakenly
# turned it on else where.
if options['fileform'] not in ("epub", "html"):
configuration.set("overrides", "include_images", "false")
adapter = adapters.getAdapter(configuration, book['url'])
adapter.is_adult = book['is_adult']
adapter.username = book['username']
adapter.password = book['password']
adapter.setChaptersRange(book['begin'], book['end'])
configuration.load_cookiejar(options['cookiejarfile'])
#logger.debug("cookiejar:%s"%configuration.cookiejar)
configuration.set_pagecache(options['pagecache'])
story = adapter.getStoryMetadataOnly()
if not story.getMetadata("series") and 'calibre_series' in book:
adapter.setSeries(book['calibre_series'][0],
book['calibre_series'][1])
# set PI version instead of default.
if 'version' in options:
story.setMetadata('version', options['version'])
book['title'] = story.getMetadata("title", removeallentities=True)
book['author_sort'] = book['author'] = story.getList(
"author", removeallentities=True)
book['publisher'] = story.getMetadata("publisher")
book['url'] = story.getMetadata("storyUrl", removeallentities=True)
book['tags'] = story.getSubjectTags(removeallentities=True)
book['comments'] = story.get_sanitized_description()
book['series'] = story.getMetadata("series",
removeallentities=True)
if story.getMetadataRaw('datePublished'):
book['pubdate'] = story.getMetadataRaw(
'datePublished').replace(tzinfo=local_tz)
if story.getMetadataRaw('dateUpdated'):
book['updatedate'] = story.getMetadataRaw(
'dateUpdated').replace(tzinfo=local_tz)
if story.getMetadataRaw('dateCreated'):
book['timestamp'] = story.getMetadataRaw(
'dateCreated').replace(tzinfo=local_tz)
else:
book['timestamp'] = datetime.now().replace(
tzinfo=local_tz) # need *something* there for calibre.
writer = writers.getWriter(options['fileform'], configuration,
adapter)
outfile = book['outfile']
## checks were done earlier, it's new or not dup or newer--just write it.
if book['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
('epub_for_update' not in book and book['collision'] in (UPDATE, UPDATEALWAYS)):
# preserve logfile even on overwrite.
if 'epub_for_update' in book:
adapter.logfile = get_update_data(
book['epub_for_update'])[6]
# change the existing entries id to notid so
# write_epub writes a whole new set to indicate overwrite.
if adapter.logfile:
adapter.logfile = adapter.logfile.replace(
"span id", "span notid")
if book['collision'] == OVERWRITE and 'fileupdated' in book:
lastupdated = story.getMetadataRaw('dateUpdated')
fileupdated = book['fileupdated']
# updated doesn't have time (or is midnight), use dates only.
# updated does have time, use full timestamps.
if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
(lastupdated.time() != time.min and fileupdated > lastupdated):
raise NotGoingToDownload(
_("Not Overwriting, web site is not newer."),
'edit-undo.png',
showerror=False)
logger.info("write to %s" % outfile)
inject_cal_cols(book, story, configuration)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = _('Download %s completed, %s chapters.') % (
options['fileform'], story.getMetadata("numChapters"))
book['all_metadata'] = story.getAllMetadata(
removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
## checks were done earlier, just update it.
elif 'epub_for_update' in book and book['collision'] in (
UPDATE, UPDATEALWAYS):
# update now handled by pre-populating the old images and
# chapters in the adapter rather than merging epubs.
#urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
# returns int adjusted for start-end range.
urlchaptercount = story.getChapterCount()
(url, chaptercount, adapter.oldchapters, adapter.oldimgs,
adapter.oldcover, adapter.calibrebookmark, adapter.logfile,
adapter.oldchaptersmap,
adapter.oldchaptersdata) = get_update_data(
book['epub_for_update'])[0:9]
# dup handling from fff_plugin needed for anthology updates.
if book['collision'] == UPDATE:
if chaptercount == urlchaptercount:
if merge:
book['comment'] = _(
"Already contains %d chapters. Reuse as is."
) % chaptercount
book['all_metadata'] = story.getAllMetadata(
removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata(
)
book['outfile'] = book[
'epub_for_update'] # for anthology merge ops.
return book
else: # not merge,
raise NotGoingToDownload(
_("Already contains %d chapters.") %
chaptercount,
'edit-undo.png',
showerror=False)
elif chaptercount > urlchaptercount:
raise NotGoingToDownload(
_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update."
) % (chaptercount, urlchaptercount),
'dialog_error.png')
elif chaptercount == 0:
raise NotGoingToDownload(
_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."
), 'dialog_error.png')
if not (book['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
and adapter.getConfig("do_update_hook"):
chaptercount = adapter.hookForUpdates(chaptercount)
logger.info("Do update - epub(%d) vs url(%d)" %
(chaptercount, urlchaptercount))
logger.info("write to %s" % outfile)
inject_cal_cols(book, story, configuration)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
(options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
book['all_metadata'] = story.getAllMetadata(
removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
else:
## Shouldn't ever get here, but hey, it happened once
## before with prefs['collision']
raise Exception(
"Impossible state reached -- Book: %s:\nOptions:%s:" %
(book, options))
if options['do_wordcount'] == SAVE_YES or (
options['do_wordcount'] == SAVE_YES_UNLESS_SITE
and not story.getMetadataRaw('numWords')):
wordcount = get_word_count(outfile)
logger.info("get_word_count:%s" % wordcount)
story.setMetadata('numWords', wordcount)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['all_metadata'] = story.getAllMetadata(
removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
if options['smarten_punctuation'] and options['fileform'] == "epub" \
and calibre_version >= (0, 9, 39):
# for smarten punc
from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
from calibre.utils.logging import Log
from collections import namedtuple
# do smarten_punctuation from calibre's polish feature
data = {'smarten_punctuation': True}
opts = ALL_OPTS.copy()
opts.update(data)
O = namedtuple('Options', ' '.join(six.iterkeys(ALL_OPTS)))
opts = O(**opts)
log = Log(level=Log.DEBUG)
polish({outfile: outfile}, opts, log, logger.info)
except NotGoingToDownload as d:
book['good'] = False
book['status'] = _('Bad')
book['showerror'] = d.showerror
book['comment'] = unicode(d)
book['icon'] = d.icon
except Exception as e:
book['good'] = False
book['status'] = _('Error')
book['comment'] = unicode(e)
book['icon'] = 'dialog_error.png'
book['status'] = _('Error')
logger.info("Exception: %s:%s" % (book, book['comment']),
exc_info=True)
#time.sleep(10)
return book
def do_download(arg, options, passed_defaultsini, passed_personalini):
# Attempt to update an existing epub.
chaptercount = None
output_filename = None
if options.unnew:
# remove mark_new_chapters marks
reset_orig_chapters_epub(arg, arg)
return
if options.update:
try:
url, chaptercount = get_dcsource_chaptercount(arg)
if not url:
print('No story URL found in epub to update.')
return
print('Updating %s, URL: %s' % (arg, url))
output_filename = arg
except Exception:
# if there's an error reading the update file, maybe it's a URL?
# we'll look for an existing outputfile down below.
url = arg
else:
url = arg
configuration = get_configuration(url, passed_defaultsini,
passed_personalini, options,
chaptercount, output_filename)
try:
# Allow chapter range with URL.
# like test1.com?sid=5[4-6] or [4,6]
# Overrides CLI options if present.
url, ch_begin, ch_end = adapters.get_url_chapter_range(url)
adapter = adapters.getAdapter(configuration, url)
## Share pagecache and cookiejar between multiple downloads.
if not hasattr(options, 'pagecache'):
options.pagecache = configuration.get_empty_pagecache()
if not hasattr(options, 'cookiejar'):
options.cookiejar = configuration.get_empty_cookiejar()
if options.save_cache:
save_cache = global_cache
save_cookies = global_cookies
else:
save_cache = save_cookies = None
configuration.set_pagecache(options.pagecache, save_cache)
configuration.set_cookiejar(options.cookiejar, save_cookies)
# url[begin-end] overrides CLI option if present.
if ch_begin or ch_end:
adapter.setChaptersRange(ch_begin, ch_end)
else:
adapter.setChaptersRange(options.begin, options.end)
# check for updating from URL (vs from file)
update_story = options.update
if update_story and not chaptercount:
try:
writer = writers.getWriter('epub', configuration, adapter)
output_filename = writer.getOutputFileName()
noturl, chaptercount = get_dcsource_chaptercount(
output_filename)
print('Updating %s, URL: %s' % (output_filename, url))
except Exception as e:
print(
"Failed to read epub for update: (%s) Continuing with update=false"
% e)
update_story = False
# Check for include_images without no_image_processing. In absence of PIL, give warning.
if adapter.getConfig('include_images') and not adapter.getConfig(
'no_image_processing'):
try:
from calibre.utils.magick import Image
except ImportError:
try:
## Pillow is a more current fork of PIL library
from PIL import Image
except ImportError:
try:
import Image
except ImportError:
print(
"You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
)
if options.interactive:
if not sys.stdin.readline().strip().lower(
).startswith('y'):
return
else:
# for non-interactive, default the response to yes and continue processing
print('y')
# three tries, that's enough if both user/pass & is_adult needed,
# or a couple tries of one or the other
for x in range(0, 2):
try:
adapter.getStoryMetadataOnly()
except exceptions.FailedToLogin as f:
if not options.interactive:
print(
'Login Failed on non-interactive process. Set username and password in personal.ini.'
)
return
if f.passwdonly:
print('Story requires a password.')
else:
print('Login Failed, Need Username/Password.')
sys.stdout.write('Username: '******'Password: '******'Login: `%s`, Password: `%s`' % (adapter.username, adapter.password))
except exceptions.AdultCheckRequired:
if options.interactive:
print(
'Please confirm you are an adult in your locale: (y/n)?'
)
if sys.stdin.readline().strip().lower().startswith('y'):
adapter.is_adult = True
else:
print(
'Adult check required on non-interactive process. Set is_adult:true in personal.ini or pass -o "is_adult=true" to the command.'
)
return
if update_story and not options.force:
urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata(
'numChapters').replace(',', ''))
# returns int adjusted for start-end range.
urlchaptercount = adapter.getStoryMetadataOnly().getChapterCount()
if chaptercount == urlchaptercount and not options.metaonly and not options.updatealways:
print('%s already contains %d chapters.' %
(output_filename, chaptercount))
elif chaptercount > urlchaptercount:
print('%s contains %d chapters, more than source: %d.' %
(output_filename, chaptercount, urlchaptercount))
elif chaptercount == 0:
print(
"%s doesn't contain any recognizable chapters, probably from a different source. Not updating."
% output_filename)
else:
# update now handled by pre-populating the old
# images and chapters in the adapter rather than
# merging epubs.
(url, chaptercount, adapter.oldchapters, adapter.oldimgs,
adapter.oldcover, adapter.calibrebookmark, adapter.logfile,
adapter.oldchaptersmap, adapter.oldchaptersdata) = (
get_update_data(output_filename))[0:9]
print('Do update - epub(%d) vs url(%d)' %
(chaptercount, urlchaptercount))
if not update_story and chaptercount == urlchaptercount and adapter.getConfig(
'do_update_hook'):
adapter.hookForUpdates(chaptercount)
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(
pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(
adapter.getConfig('pre_process_cmd')).substitute(metadata),
shell=True)
output_filename = write_story(configuration,
adapter,
'epub',
nooutput=options.nooutput)
else:
if not options.metaonly and adapter.getConfig('pre_process_cmd'):
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(
pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(
adapter.getConfig('pre_process_cmd')).substitute(metadata),
shell=True)
output_filename = write_story(configuration,
adapter,
options.format,
metaonly=options.metaonly,
nooutput=options.nooutput)
if options.metaonly and not options.jsonmeta:
metadata = adapter.getStoryMetadataOnly().getAllMetadata()
metadata['output_filename'] = output_filename
if not options.nometachapters:
metadata['zchapters'] = []
for i, chap in enumerate(adapter.get_chapters()):
metadata['zchapters'].append((i + 1, chap))
pprint.pprint(metadata)
if not options.metaonly and adapter.getConfig('post_process_cmd'):
if adapter.getConfig('post_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(
pattern=adapter.getConfig('post_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
metadata['output_filename'] = output_filename
call(string.Template(
adapter.getConfig('post_process_cmd')).substitute(metadata),
shell=True)
if options.jsonmeta:
metadata = adapter.getStoryMetadataOnly().getAllMetadata()
metadata['output_filename'] = output_filename
if not options.nometachapters:
metadata['zchapters'] = []
for i, chap in enumerate(adapter.get_chapters()):
metadata['zchapters'].append((i + 1, chap))
import json
print(
json.dumps(metadata,
sort_keys=True,
indent=2,
separators=(',', ':')))
del adapter
except exceptions.InvalidStoryURL as isu:
print(isu)
except exceptions.StoryDoesNotExist as dne:
print(dne)
except exceptions.UnknownSite as us:
print(us)
except exceptions.AccessDenied as ad:
print(ad)
def do_download_for_worker(book,options,merge,notification=lambda x,y:x):
'''
Child job, to download story when run as a worker job
'''
from calibre_plugins.fanficfare_plugin import FanFicFareBase
fffbase = FanFicFareBase(options['plugin_path'])
with fffbase:
from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload,
OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY, CALIBREONLYSAVECOL)
from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
from calibre_plugins.fanficfare_plugin.fff_util import (get_fff_adapter, get_fff_config)
try:
book['comment'] = _('Download started...')
configuration = get_fff_config(book['url'],
options['fileform'],
options['personal.ini'])
if configuration.getConfig('use_ssl_unverified_context'):
## monkey patch to avoid SSL bug. dupliated from
## fff_plugin.py because bg jobs run in own process
## space.
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
configuration.set("overrides","never_make_cover","true")
# images only for epub, html, even if the user mistakenly
# turned it on else where.
if options['fileform'] not in ("epub","html"):
configuration.set("overrides","include_images","false")
adapter = adapters.getAdapter(configuration,book['url'])
adapter.is_adult = book['is_adult']
adapter.username = book['username']
adapter.password = book['password']
adapter.setChaptersRange(book['begin'],book['end'])
adapter.load_cookiejar(options['cookiejarfile'])
#logger.debug("cookiejar:%s"%adapter.cookiejar)
adapter.set_pagecache(options['pagecache'])
story = adapter.getStoryMetadataOnly()
if 'calibre_series' in book:
adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
# set PI version instead of default.
if 'version' in options:
story.setMetadata('version',options['version'])
book['title'] = story.getMetadata("title", removeallentities=True)
book['author_sort'] = book['author'] = story.getList("author", removeallentities=True)
book['publisher'] = story.getMetadata("site")
book['url'] = story.getMetadata("storyUrl")
book['tags'] = story.getSubjectTags(removeallentities=True)
if story.getMetadata("description"):
book['comments'] = sanitize_comments_html(story.getMetadata("description"))
else:
book['comments']=''
book['series'] = story.getMetadata("series", removeallentities=True)
if story.getMetadataRaw('datePublished'):
book['pubdate'] = story.getMetadataRaw('datePublished').replace(tzinfo=local_tz)
if story.getMetadataRaw('dateUpdated'):
book['updatedate'] = story.getMetadataRaw('dateUpdated').replace(tzinfo=local_tz)
if story.getMetadataRaw('dateCreated'):
book['timestamp'] = story.getMetadataRaw('dateCreated').replace(tzinfo=local_tz)
else:
book['timestamp'] = None # need *something* there for calibre.
writer = writers.getWriter(options['fileform'],configuration,adapter)
outfile = book['outfile']
## No need to download at all. Shouldn't ever get down here.
if options['collision'] in (CALIBREONLY, CALIBREONLYSAVECOL):
logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
book['comment'] = 'Metadata collected.'
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
## checks were done earlier, it's new or not dup or newer--just write it.
elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
# preserve logfile even on overwrite.
if 'epub_for_update' in book:
adapter.logfile = get_update_data(book['epub_for_update'])[6]
# change the existing entries id to notid so
# write_epub writes a whole new set to indicate overwrite.
if adapter.logfile:
adapter.logfile = adapter.logfile.replace("span id","span notid")
if options['collision'] == OVERWRITE and 'fileupdated' in book:
lastupdated=story.getMetadataRaw('dateUpdated')
fileupdated=book['fileupdated']
# updated doesn't have time (or is midnight), use dates only.
# updated does have time, use full timestamps.
if (lastupdated.time() == time.min and fileupdated.date() > lastupdated.date()) or \
(lastupdated.time() != time.min and fileupdated > lastupdated):
raise NotGoingToDownload(_("Not Overwriting, web site is not newer."),'edit-undo.png')
logger.info("write to %s"%outfile)
inject_cal_cols(book,story,configuration)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
## checks were done earlier, just update it.
elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
# update now handled by pre-populating the old images and
# chapters in the adapter rather than merging epubs.
urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
(url,
chaptercount,
adapter.oldchapters,
adapter.oldimgs,
adapter.oldcover,
adapter.calibrebookmark,
adapter.logfile,
adapter.oldchaptersmap,
adapter.oldchaptersdata) = get_update_data(book['epub_for_update'])[0:9]
# dup handling from fff_plugin needed for anthology updates.
if options['collision'] == UPDATE:
if chaptercount == urlchaptercount:
if merge:
book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
book['outfile'] = book['epub_for_update'] # for anthology merge ops.
return book
else: # not merge,
raise NotGoingToDownload(_("Already contains %d chapters.")%chaptercount,'edit-undo.png')
elif chaptercount > urlchaptercount:
raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
elif chaptercount == 0:
raise NotGoingToDownload(_("FanFicFare doesn't recognize chapters in existing epub, epub is probably from a different source. Use Overwrite to force update."),'dialog_error.png')
if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
and adapter.getConfig("do_update_hook"):
chaptercount = adapter.hookForUpdates(chaptercount)
logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
logger.info("write to %s"%outfile)
inject_cal_cols(book,story,configuration)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
(options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
if options['do_wordcount'] == SAVE_YES or (
options['do_wordcount'] == SAVE_YES_UNLESS_SITE and not story.getMetadataRaw('numWords') ):
wordcount = get_word_count(outfile)
logger.info("get_word_count:%s"%wordcount)
story.setMetadata('numWords',wordcount)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['all_metadata'] = story.getAllMetadata(removeallentities=True)
if options['savemetacol'] != '':
book['savemetacol'] = story.dump_html_metadata()
if options['smarten_punctuation'] and options['fileform'] == "epub" \
and calibre_version >= (0, 9, 39):
# for smarten punc
from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
from calibre.utils.logging import Log
from collections import namedtuple
# do smarten_punctuation from calibre's polish feature
data = {'smarten_punctuation':True}
opts = ALL_OPTS.copy()
opts.update(data)
O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
opts = O(**opts)
log = Log(level=Log.DEBUG)
polish({outfile:outfile}, opts, log, logger.info)
except NotGoingToDownload as d:
book['good']=False
book['comment']=unicode(d)
book['icon'] = d.icon
except Exception as e:
book['good']=False
book['comment']=unicode(e)
book['icon']='dialog_error.png'
book['status'] = 'Error'
logger.info("Exception: %s:%s"%(book,unicode(e)))
traceback.print_exc()
#time.sleep(10)
return book
def do_download(arg,
options,
passed_defaultsini,
passed_personalini):
# Attempt to update an existing epub.
chaptercount = None
output_filename = None
if options.unnew:
# remove mark_new_chapters marks
reset_orig_chapters_epub(arg,arg)
return
if options.update:
try:
url, chaptercount = get_dcsource_chaptercount(arg)
if not url:
print('No story URL found in epub to update.')
return
print('Updating %s, URL: %s' % (arg, url))
output_filename = arg
except Exception:
# if there's an error reading the update file, maybe it's a URL?
# we'll look for an existing outputfile down below.
url = arg
else:
url = arg
configuration = get_configuration(url,
passed_defaultsini,
passed_personalini,
options,
chaptercount,
output_filename)
try:
# Allow chapter range with URL.
# like test1.com?sid=5[4-6] or [4,6]
# Overrides CLI options if present.
url,ch_begin,ch_end = adapters.get_url_chapter_range(url)
adapter = adapters.getAdapter(configuration, url)
## Share pagecache and cookiejar between multiple downloads.
if not hasattr(options,'pagecache'):
options.pagecache = configuration.get_empty_pagecache()
if not hasattr(options,'cookiejar'):
options.cookiejar = configuration.get_empty_cookiejar()
if options.save_cache:
save_cache = global_cache
save_cookies = global_cookies
else:
save_cache = save_cookies = None
configuration.set_pagecache(options.pagecache,save_cache)
configuration.set_cookiejar(options.cookiejar,save_cookies)
# url[begin-end] overrides CLI option if present.
if ch_begin or ch_end:
adapter.setChaptersRange(ch_begin, ch_end)
else:
adapter.setChaptersRange(options.begin, options.end)
# check for updating from URL (vs from file)
if options.update and not chaptercount:
try:
writer = writers.getWriter('epub', configuration, adapter)
output_filename = writer.getOutputFileName()
noturl, chaptercount = get_dcsource_chaptercount(output_filename)
print('Updating %s, URL: %s' % (output_filename, url))
except Exception as e:
print("Failed to read epub for update: (%s) Continuing with update=false"%e)
options.update = False
# Check for include_images without no_image_processing. In absence of PIL, give warning.
if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
try:
from calibre.utils.magick import Image
except ImportError:
try:
## Pillow is a more current fork of PIL library
from PIL import Image
except ImportError:
try:
import Image
except ImportError:
print("You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?")
if options.interactive:
if not sys.stdin.readline().strip().lower().startswith('y'):
return
else:
# for non-interactive, default the response to yes and continue processing
print('y')
# three tries, that's enough if both user/pass & is_adult needed,
# or a couple tries of one or the other
for x in range(0, 2):
try:
adapter.getStoryMetadataOnly()
except exceptions.FailedToLogin as f:
if not options.interactive:
print('Login Failed on non-interactive process. Set username and password in personal.ini.')
return
if f.passwdonly:
print('Story requires a password.')
else:
print('Login Failed, Need Username/Password.')
sys.stdout.write('Username: '******'Password: '******'Login: `%s`, Password: `%s`' % (adapter.username, adapter.password))
except exceptions.AdultCheckRequired:
if options.interactive:
print('Please confirm you are an adult in your locale: (y/n)?')
if sys.stdin.readline().strip().lower().startswith('y'):
adapter.is_adult = True
else:
print('Adult check required on non-interactive process. Set is_adult:true in personal.ini or pass -o "is_adult=true" to the command.')
return
if options.update and not options.force:
urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters').replace(',',''))
# returns int adjusted for start-end range.
urlchaptercount = adapter.getStoryMetadataOnly().getChapterCount()
if chaptercount == urlchaptercount and not options.metaonly:
print('%s already contains %d chapters.' % (output_filename, chaptercount))
elif chaptercount > urlchaptercount:
print('%s contains %d chapters, more than source: %d.' % (output_filename, chaptercount, urlchaptercount))
elif chaptercount == 0:
print("%s doesn't contain any recognizable chapters, probably from a different source. Not updating." % output_filename)
else:
# update now handled by pre-populating the old
# images and chapters in the adapter rather than
# merging epubs.
(url,
chaptercount,
adapter.oldchapters,
adapter.oldimgs,
adapter.oldcover,
adapter.calibrebookmark,
adapter.logfile,
adapter.oldchaptersmap,
adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]
print('Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount))
if not options.update and chaptercount == urlchaptercount and adapter.getConfig('do_update_hook'):
adapter.hookForUpdates(chaptercount)
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(adapter.getConfig('pre_process_cmd')).substitute(metadata), shell=True)
write_story(configuration, adapter, 'epub')
else:
# regular download
if options.metaonly:
metadata = adapter.getStoryMetadataOnly().getAllMetadata()
if not options.nometachapters:
metadata['zchapters'] = []
for i, chap in enumerate(adapter.get_chapters()):
metadata['zchapters'].append((i+1,chap))
if not options.metaonly and adapter.getConfig('pre_process_cmd'):
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(adapter.getConfig('pre_process_cmd')).substitute(metadata), shell=True)
output_filename = write_story(configuration, adapter, options.format, options.metaonly)
if options.metaonly:
metadata['output_filename'] = output_filename
if options.jsonmeta:
import json
print(json.dumps(metadata, sort_keys=True,
indent=2, separators=(',', ':')))
else:
pprint.pprint(metadata)
if not options.metaonly and adapter.getConfig('post_process_cmd'):
if adapter.getConfig('post_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('post_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
metadata['output_filename'] = output_filename
call(string.Template(adapter.getConfig('post_process_cmd')).substitute(metadata), shell=True)
del adapter
except exceptions.InvalidStoryURL as isu:
print(isu)
except exceptions.StoryDoesNotExist as dne:
print(dne)
except exceptions.UnknownSite as us:
print(us)
except exceptions.AccessDenied as ad:
print(ad)
if chaptercount == urlchaptercount and not options.metaonly:
print '%s already contains %d chapters.' % (output_filename,
chaptercount)
elif chaptercount > urlchaptercount:
print '%s contains %d chapters, more than source: %d.' % (
output_filename, chaptercount, urlchaptercount)
elif chaptercount == 0:
print "%s doesn't contain any recognizable chapters, probably from a different source. Not updating." % output_filename
else:
# update now handled by pre-populating the old
# images and chapters in the adapter rather than
# merging epubs.
(url, chaptercount, adapter.oldchapters, adapter.oldimgs,
adapter.oldcover, adapter.calibrebookmark, adapter.logfile,
adapter.oldchaptersmap, adapter.oldchaptersdata) = (
get_update_data(output_filename))[0:9]
print 'Do update - epub(%d) vs url(%d)' % (chaptercount,
urlchaptercount)
if not options.update and chaptercount == urlchaptercount and adapter.getConfig(
'do_update_hook'):
adapter.hookForUpdates(chaptercount)
write_story(configuration, adapter, 'epub')
else:
# regular download
if options.metaonly:
metadata = adapter.getStoryMetadataOnly().getAllMetadata()
metadata['zchapters'] = []
def do_download_for_worker(book, options, notification=lambda x, y: x):
'''
Child job, to download story when run as a worker job
'''
from calibre_plugins.fanficfare_plugin import FanFicFareBase
fffbase = FanFicFareBase(options['plugin_path'])
with fffbase:
from calibre_plugins.fanficfare_plugin.dialogs import (
NotGoingToDownload, OVERWRITE, OVERWRITEALWAYS, UPDATE,
UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
from calibre_plugins.fanficfare_plugin.fff_util import (
get_fff_adapter, get_fff_config)
try:
book['comment'] = _('Download started...')
configuration = get_fff_config(book['url'], options['fileform'],
options['personal.ini'])
if configuration.getConfig('use_ssl_unverified_context'):
## monkey patch to avoid SSL bug. dupliated from
## fff_plugin.py because bg jobs run in own process
## space.
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
if not options[
'updateepubcover'] and 'epub_for_update' in book and options[
'collision'] in (UPDATE, UPDATEALWAYS):
configuration.set("overrides", "never_make_cover", "true")
# images only for epub, html, even if the user mistakenly
# turned it on else where.
if options['fileform'] not in ("epub", "html"):
configuration.set("overrides", "include_images", "false")
adapter = adapters.getAdapter(configuration, book['url'])
adapter.is_adult = book['is_adult']
adapter.username = book['username']
adapter.password = book['password']
adapter.setChaptersRange(book['begin'], book['end'])
adapter.load_cookiejar(options['cookiejarfile'])
logger.debug("cookiejar:%s" % adapter.cookiejar)
adapter.set_pagecache(options['pagecache'])
story = adapter.getStoryMetadataOnly()
if 'calibre_series' in book:
adapter.setSeries(book['calibre_series'][0],
book['calibre_series'][1])
# set PI version instead of default.
if 'version' in options:
story.setMetadata('version', options['version'])
writer = writers.getWriter(options['fileform'], configuration,
adapter)
outfile = book['outfile']
## No need to download at all. Shouldn't ever get down here.
if options['collision'] in (CALIBREONLY):
logger.info(
"Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening..."
)
book['comment'] = 'Metadata collected.'
## checks were done earlier, it's new or not dup or newer--just write it.
elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
# preserve logfile even on overwrite.
if 'epub_for_update' in book:
(
urlignore,
chaptercountignore,
oldchaptersignore,
oldimgsignore,
oldcoverignore,
calibrebookmarkignore,
# only logfile set in adapter, so others aren't used.
adapter.logfile) = get_update_data(
book['epub_for_update'])
# change the existing entries id to notid so
# write_epub writes a whole new set to indicate overwrite.
if adapter.logfile:
adapter.logfile = adapter.logfile.replace(
"span id", "span notid")
logger.info("write to %s" % outfile)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = 'Download %s completed, %s chapters.' % (
options['fileform'], story.getMetadata("numChapters"))
## checks were done earlier, just update it.
elif 'epub_for_update' in book and options['collision'] in (
UPDATE, UPDATEALWAYS):
# update now handled by pre-populating the old images and
# chapters in the adapter rather than merging epubs.
urlchaptercount = int(
story.getMetadata('numChapters').replace(',', ''))
(url, chaptercount, adapter.oldchapters, adapter.oldimgs,
adapter.oldcover, adapter.calibrebookmark,
adapter.logfile) = get_update_data(book['epub_for_update'])
# dup handling from fff_plugin needed for anthology updates.
if options['collision'] == UPDATE:
if chaptercount == urlchaptercount:
book['comment'] = _(
"Already contains %d chapters. Reuse as is."
) % chaptercount
book['outfile'] = book[
'epub_for_update'] # for anthology merge ops.
return book
# dup handling from fff_plugin needed for anthology updates.
if chaptercount > urlchaptercount:
raise NotGoingToDownload(
_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update."
) % (chaptercount, urlchaptercount),
'dialog_error.png')
if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
and adapter.getConfig("do_update_hook"):
chaptercount = adapter.hookForUpdates(chaptercount)
logger.info("Do update - epub(%d) vs url(%d)" %
(chaptercount, urlchaptercount))
logger.info("write to %s" % outfile)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
(options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
if options['smarten_punctuation'] and options['fileform'] == "epub" \
and calibre_version >= (0, 9, 39):
# for smarten punc
from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
from calibre.utils.logging import Log
from collections import namedtuple
# do smarten_punctuation from calibre's polish feature
data = {'smarten_punctuation': True}
opts = ALL_OPTS.copy()
opts.update(data)
O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
opts = O(**opts)
log = Log(level=Log.DEBUG)
# report = []
polish({outfile: outfile}, opts, log,
logger.info) # report.append
except NotGoingToDownload as d:
book['good'] = False
book['comment'] = unicode(d)
book['icon'] = d.icon
except Exception as e:
book['good'] = False
book['comment'] = unicode(e)
book['icon'] = 'dialog_error.png'
book['status'] = 'Error'
logger.info("Exception: %s:%s" % (book, unicode(e)))
traceback.print_exc()
#time.sleep(10)
return book
print '%s contains %d chapters, more than source: %d.' % (output_filename, chaptercount, urlchaptercount)
elif chaptercount == 0:
print "%s doesn't contain any recognizable chapters, probably from a different source. Not updating." % output_filename
else:
# update now handled by pre-populating the old
# images and chapters in the adapter rather than
# merging epubs.
(url,
chaptercount,
adapter.oldchapters,
adapter.oldimgs,
adapter.oldcover,
adapter.calibrebookmark,
adapter.logfile,
adapter.oldchaptersmap,
adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]
print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)
if not options.update and chaptercount == urlchaptercount and adapter.getConfig('do_update_hook'):
adapter.hookForUpdates(chaptercount)
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(adapter.getConfig('pre_process_cmd')).substitute(metadata), shell=True)
write_story(configuration, adapter, 'epub')
else:
print '%s contains %d chapters, more than source: %d.' % (output_filename, chaptercount, urlchaptercount)
elif chaptercount == 0:
print "%s doesn't contain any recognizable chapters, probably from a different source. Not updating." % output_filename
else:
# update now handled by pre-populating the old
# images and chapters in the adapter rather than
# merging epubs.
(url,
chaptercount,
adapter.oldchapters,
adapter.oldimgs,
adapter.oldcover,
adapter.calibrebookmark,
adapter.logfile,
adapter.oldchaptersmap,
adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]
print 'Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)
if not options.update and chaptercount == urlchaptercount and adapter.getConfig('do_update_hook'):
adapter.hookForUpdates(chaptercount)
write_story(configuration, adapter, 'epub')
else:
# regular download
if options.metaonly:
pprint.pprint(adapter.getStoryMetadataOnly().getAllMetadata())
pprint.pprint(adapter.chapterUrls)
output_filename = write_story(configuration, adapter, options.format, options.metaonly)
def do_download_for_worker(book,options,notification=lambda x,y:x):
'''
Child job, to download story when run as a worker job
'''
from calibre_plugins.fanficfare_plugin import FanFicFareBase
fffbase = FanFicFareBase(options['plugin_path'])
with fffbase:
from calibre_plugins.fanficfare_plugin.dialogs import (NotGoingToDownload,
OVERWRITE, OVERWRITEALWAYS, UPDATE, UPDATEALWAYS, ADDNEW, SKIP, CALIBREONLY)
from calibre_plugins.fanficfare_plugin.fanficfare import adapters, writers, exceptions
from calibre_plugins.fanficfare_plugin.fanficfare.epubutils import get_update_data
from calibre_plugins.fanficfare_plugin.fff_util import (get_fff_adapter, get_fff_config)
try:
book['comment'] = _('Download started...')
configuration = get_fff_config(book['url'],
options['fileform'],
options['personal.ini'])
if configuration.getConfig('use_ssl_unverified_context'):
## monkey patch to avoid SSL bug. dupliated from
## fff_plugin.py because bg jobs run in own process
## space.
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
if not options['updateepubcover'] and 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
configuration.set("overrides","never_make_cover","true")
# images only for epub, html, even if the user mistakenly
# turned it on else where.
if options['fileform'] not in ("epub","html"):
configuration.set("overrides","include_images","false")
adapter = adapters.getAdapter(configuration,book['url'])
adapter.is_adult = book['is_adult']
adapter.username = book['username']
adapter.password = book['password']
adapter.setChaptersRange(book['begin'],book['end'])
adapter.load_cookiejar(options['cookiejarfile'])
logger.debug("cookiejar:%s"%adapter.cookiejar)
adapter.set_pagecache(options['pagecache'])
story = adapter.getStoryMetadataOnly()
if 'calibre_series' in book:
adapter.setSeries(book['calibre_series'][0],book['calibre_series'][1])
# set PI version instead of default.
if 'version' in options:
story.setMetadata('version',options['version'])
writer = writers.getWriter(options['fileform'],configuration,adapter)
outfile = book['outfile']
## No need to download at all. Shouldn't ever get down here.
if options['collision'] in (CALIBREONLY):
logger.info("Skipping CALIBREONLY 'update' down inside worker--this shouldn't be happening...")
book['comment'] = 'Metadata collected.'
## checks were done earlier, it's new or not dup or newer--just write it.
elif options['collision'] in (ADDNEW, SKIP, OVERWRITE, OVERWRITEALWAYS) or \
('epub_for_update' not in book and options['collision'] in (UPDATE, UPDATEALWAYS)):
# preserve logfile even on overwrite.
if 'epub_for_update' in book:
(urlignore,
chaptercountignore,
oldchaptersignore,
oldimgsignore,
oldcoverignore,
calibrebookmarkignore,
# only logfile set in adapter, so others aren't used.
adapter.logfile) = get_update_data(book['epub_for_update'])
# change the existing entries id to notid so
# write_epub writes a whole new set to indicate overwrite.
if adapter.logfile:
adapter.logfile = adapter.logfile.replace("span id","span notid")
logger.info("write to %s"%outfile)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = 'Download %s completed, %s chapters.'%(options['fileform'],story.getMetadata("numChapters"))
## checks were done earlier, just update it.
elif 'epub_for_update' in book and options['collision'] in (UPDATE, UPDATEALWAYS):
# update now handled by pre-populating the old images and
# chapters in the adapter rather than merging epubs.
urlchaptercount = int(story.getMetadata('numChapters').replace(',',''))
(url,
chaptercount,
adapter.oldchapters,
adapter.oldimgs,
adapter.oldcover,
adapter.calibrebookmark,
adapter.logfile) = get_update_data(book['epub_for_update'])
# dup handling from fff_plugin needed for anthology updates.
if options['collision'] == UPDATE:
if chaptercount == urlchaptercount:
book['comment']=_("Already contains %d chapters. Reuse as is.")%chaptercount
book['outfile'] = book['epub_for_update'] # for anthology merge ops.
return book
# dup handling from fff_plugin needed for anthology updates.
if chaptercount > urlchaptercount:
raise NotGoingToDownload(_("Existing epub contains %d chapters, web site only has %d. Use Overwrite to force update.") % (chaptercount,urlchaptercount),'dialog_error.png')
if not (options['collision'] == UPDATEALWAYS and chaptercount == urlchaptercount) \
and adapter.getConfig("do_update_hook"):
chaptercount = adapter.hookForUpdates(chaptercount)
logger.info("Do update - epub(%d) vs url(%d)" % (chaptercount, urlchaptercount))
logger.info("write to %s"%outfile)
writer.writeStory(outfilename=outfile, forceOverwrite=True)
book['comment'] = _('Update %s completed, added %s chapters for %s total.')%\
(options['fileform'],(urlchaptercount-chaptercount),urlchaptercount)
if options['smarten_punctuation'] and options['fileform'] == "epub" \
and calibre_version >= (0, 9, 39):
# for smarten punc
from calibre.ebooks.oeb.polish.main import polish, ALL_OPTS
from calibre.utils.logging import Log
from collections import namedtuple
# do smarten_punctuation from calibre's polish feature
data = {'smarten_punctuation':True}
opts = ALL_OPTS.copy()
opts.update(data)
O = namedtuple('Options', ' '.join(ALL_OPTS.iterkeys()))
opts = O(**opts)
log = Log(level=Log.DEBUG)
# report = []
polish({outfile:outfile}, opts, log, logger.info) # report.append
except NotGoingToDownload as d:
book['good']=False
book['comment']=unicode(d)
book['icon'] = d.icon
except Exception as e:
book['good']=False
book['comment']=unicode(e)
book['icon']='dialog_error.png'
book['status'] = 'Error'
logger.info("Exception: %s:%s"%(book,unicode(e)))
traceback.print_exc()
#time.sleep(10)
return book