def do_download(arg, options, passed_defaultsini, passed_personalini):
# Attempt to update an existing epub.
chaptercount = None
output_filename = None
if options.unnew:
# remove mark_new_chapters marks
reset_orig_chapters_epub(arg, arg)
return
if options.update:
try:
url, chaptercount = get_dcsource_chaptercount(arg)
if not url:
print 'No story URL found in epub to update.'
return
print 'Updating %s, URL: %s' % (arg, url)
output_filename = arg
except Exception:
# if there's an error reading the update file, maybe it's a URL?
# we'll look for an existing outputfile down below.
url = arg
else:
url = arg
try:
configuration = Configuration(adapters.getConfigSectionsFor(url),
options.format)
except exceptions.UnknownSite, e:
if options.list or options.normalize:
# list for page doesn't have to be a supported site.
configuration = Configuration('test1.com', options.format)
else:
raise e
def do_download(arg,
options,
passed_defaultsini,
passed_personalini):
# Attempt to update an existing epub.
chaptercount = None
output_filename = None
if options.unnew:
# remove mark_new_chapters marks
reset_orig_chapters_epub(arg,arg)
return
if options.update:
try:
url, chaptercount = get_dcsource_chaptercount(arg)
if not url:
print 'No story URL found in epub to update.'
return
print 'Updating %s, URL: %s' % (arg, url)
output_filename = arg
except Exception:
# if there's an error reading the update file, maybe it's a URL?
# we'll look for an existing outputfile down below.
url = arg
else:
url = arg
try:
configuration = Configuration(adapters.getConfigSectionsFor(url), options.format)
except exceptions.UnknownSite, e:
if options.list or options.normalize:
# list for page doesn't have to be a supported site.
configuration = Configuration('test1.com', options.format)
else:
raise e
def do_download(arg, options, passed_defaultsini, passed_personalini):
# Attempt to update an existing epub.
chaptercount = None
output_filename = None
if options.unnew:
# remove mark_new_chapters marks
reset_orig_chapters_epub(arg, arg)
return
if options.update:
try:
url, chaptercount = get_dcsource_chaptercount(arg)
if not url:
print('No story URL found in epub to update.')
return
print('Updating %s, URL: %s' % (arg, url))
output_filename = arg
except Exception:
# if there's an error reading the update file, maybe it's a URL?
# we'll look for an existing outputfile down below.
url = arg
else:
url = arg
configuration = get_configuration(url, passed_defaultsini,
passed_personalini, options,
chaptercount, output_filename)
try:
# Allow chapter range with URL.
# like test1.com?sid=5[4-6] or [4,6]
# Overrides CLI options if present.
url, ch_begin, ch_end = adapters.get_url_chapter_range(url)
adapter = adapters.getAdapter(configuration, url)
## Share pagecache and cookiejar between multiple downloads.
if not hasattr(options, 'pagecache'):
options.pagecache = configuration.get_empty_pagecache()
if not hasattr(options, 'cookiejar'):
options.cookiejar = configuration.get_empty_cookiejar()
if options.save_cache:
save_cache = global_cache
save_cookies = global_cookies
else:
save_cache = save_cookies = None
configuration.set_pagecache(options.pagecache, save_cache)
configuration.set_cookiejar(options.cookiejar, save_cookies)
# url[begin-end] overrides CLI option if present.
if ch_begin or ch_end:
adapter.setChaptersRange(ch_begin, ch_end)
else:
adapter.setChaptersRange(options.begin, options.end)
# check for updating from URL (vs from file)
update_story = options.update
if update_story and not chaptercount:
try:
writer = writers.getWriter('epub', configuration, adapter)
output_filename = writer.getOutputFileName()
noturl, chaptercount = get_dcsource_chaptercount(
output_filename)
print('Updating %s, URL: %s' % (output_filename, url))
except Exception as e:
print(
"Failed to read epub for update: (%s) Continuing with update=false"
% e)
update_story = False
# Check for include_images without no_image_processing. In absence of PIL, give warning.
if adapter.getConfig('include_images') and not adapter.getConfig(
'no_image_processing'):
try:
from calibre.utils.magick import Image
except ImportError:
try:
## Pillow is a more current fork of PIL library
from PIL import Image
except ImportError:
try:
import Image
except ImportError:
print(
"You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
)
if options.interactive:
if not sys.stdin.readline().strip().lower(
).startswith('y'):
return
else:
# for non-interactive, default the response to yes and continue processing
print('y')
# three tries, that's enough if both user/pass & is_adult needed,
# or a couple tries of one or the other
for x in range(0, 2):
try:
adapter.getStoryMetadataOnly()
except exceptions.FailedToLogin as f:
if not options.interactive:
print(
'Login Failed on non-interactive process. Set username and password in personal.ini.'
)
return
if f.passwdonly:
print('Story requires a password.')
else:
print('Login Failed, Need Username/Password.')
sys.stdout.write('Username: '******'Password: '******'Login: `%s`, Password: `%s`' % (adapter.username, adapter.password))
except exceptions.AdultCheckRequired:
if options.interactive:
print(
'Please confirm you are an adult in your locale: (y/n)?'
)
if sys.stdin.readline().strip().lower().startswith('y'):
adapter.is_adult = True
else:
print(
'Adult check required on non-interactive process. Set is_adult:true in personal.ini or pass -o "is_adult=true" to the command.'
)
return
if update_story and not options.force:
urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata(
'numChapters').replace(',', ''))
# returns int adjusted for start-end range.
urlchaptercount = adapter.getStoryMetadataOnly().getChapterCount()
if chaptercount == urlchaptercount and not options.metaonly and not options.updatealways:
print('%s already contains %d chapters.' %
(output_filename, chaptercount))
elif chaptercount > urlchaptercount:
print('%s contains %d chapters, more than source: %d.' %
(output_filename, chaptercount, urlchaptercount))
elif chaptercount == 0:
print(
"%s doesn't contain any recognizable chapters, probably from a different source. Not updating."
% output_filename)
else:
# update now handled by pre-populating the old
# images and chapters in the adapter rather than
# merging epubs.
(url, chaptercount, adapter.oldchapters, adapter.oldimgs,
adapter.oldcover, adapter.calibrebookmark, adapter.logfile,
adapter.oldchaptersmap, adapter.oldchaptersdata) = (
get_update_data(output_filename))[0:9]
print('Do update - epub(%d) vs url(%d)' %
(chaptercount, urlchaptercount))
if not update_story and chaptercount == urlchaptercount and adapter.getConfig(
'do_update_hook'):
adapter.hookForUpdates(chaptercount)
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(
pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(
adapter.getConfig('pre_process_cmd')).substitute(metadata),
shell=True)
output_filename = write_story(configuration,
adapter,
'epub',
nooutput=options.nooutput)
else:
if not options.metaonly and adapter.getConfig('pre_process_cmd'):
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(
pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(
adapter.getConfig('pre_process_cmd')).substitute(metadata),
shell=True)
output_filename = write_story(configuration,
adapter,
options.format,
metaonly=options.metaonly,
nooutput=options.nooutput)
if options.metaonly and not options.jsonmeta:
metadata = adapter.getStoryMetadataOnly().getAllMetadata()
metadata['output_filename'] = output_filename
if not options.nometachapters:
metadata['zchapters'] = []
for i, chap in enumerate(adapter.get_chapters()):
metadata['zchapters'].append((i + 1, chap))
pprint.pprint(metadata)
if not options.metaonly and adapter.getConfig('post_process_cmd'):
if adapter.getConfig('post_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(
pattern=adapter.getConfig('post_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
metadata['output_filename'] = output_filename
call(string.Template(
adapter.getConfig('post_process_cmd')).substitute(metadata),
shell=True)
if options.jsonmeta:
metadata = adapter.getStoryMetadataOnly().getAllMetadata()
metadata['output_filename'] = output_filename
if not options.nometachapters:
metadata['zchapters'] = []
for i, chap in enumerate(adapter.get_chapters()):
metadata['zchapters'].append((i + 1, chap))
import json
print(
json.dumps(metadata,
sort_keys=True,
indent=2,
separators=(',', ':')))
del adapter
except exceptions.InvalidStoryURL as isu:
print(isu)
except exceptions.StoryDoesNotExist as dne:
print(dne)
except exceptions.UnknownSite as us:
print(us)
except exceptions.AccessDenied as ad:
print(ad)
def do_download(arg,
options,
passed_defaultsini,
passed_personalini):
# Attempt to update an existing epub.
chaptercount = None
output_filename = None
if options.unnew:
# remove mark_new_chapters marks
reset_orig_chapters_epub(arg,arg)
return
if options.update:
try:
url, chaptercount = get_dcsource_chaptercount(arg)
if not url:
print('No story URL found in epub to update.')
return
print('Updating %s, URL: %s' % (arg, url))
output_filename = arg
except Exception:
# if there's an error reading the update file, maybe it's a URL?
# we'll look for an existing outputfile down below.
url = arg
else:
url = arg
configuration = get_configuration(url,
passed_defaultsini,
passed_personalini,
options,
chaptercount,
output_filename)
try:
# Allow chapter range with URL.
# like test1.com?sid=5[4-6] or [4,6]
# Overrides CLI options if present.
url,ch_begin,ch_end = adapters.get_url_chapter_range(url)
adapter = adapters.getAdapter(configuration, url)
## Share pagecache and cookiejar between multiple downloads.
if not hasattr(options,'pagecache'):
options.pagecache = configuration.get_empty_pagecache()
if not hasattr(options,'cookiejar'):
options.cookiejar = configuration.get_empty_cookiejar()
if options.save_cache:
save_cache = global_cache
save_cookies = global_cookies
else:
save_cache = save_cookies = None
configuration.set_pagecache(options.pagecache,save_cache)
configuration.set_cookiejar(options.cookiejar,save_cookies)
# url[begin-end] overrides CLI option if present.
if ch_begin or ch_end:
adapter.setChaptersRange(ch_begin, ch_end)
else:
adapter.setChaptersRange(options.begin, options.end)
# check for updating from URL (vs from file)
if options.update and not chaptercount:
try:
writer = writers.getWriter('epub', configuration, adapter)
output_filename = writer.getOutputFileName()
noturl, chaptercount = get_dcsource_chaptercount(output_filename)
print('Updating %s, URL: %s' % (output_filename, url))
except Exception as e:
print("Failed to read epub for update: (%s) Continuing with update=false"%e)
options.update = False
# Check for include_images without no_image_processing. In absence of PIL, give warning.
if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
try:
from calibre.utils.magick import Image
except ImportError:
try:
## Pillow is a more current fork of PIL library
from PIL import Image
except ImportError:
try:
import Image
except ImportError:
print("You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?")
if options.interactive:
if not sys.stdin.readline().strip().lower().startswith('y'):
return
else:
# for non-interactive, default the response to yes and continue processing
print('y')
# three tries, that's enough if both user/pass & is_adult needed,
# or a couple tries of one or the other
for x in range(0, 2):
try:
adapter.getStoryMetadataOnly()
except exceptions.FailedToLogin as f:
if not options.interactive:
print('Login Failed on non-interactive process. Set username and password in personal.ini.')
return
if f.passwdonly:
print('Story requires a password.')
else:
print('Login Failed, Need Username/Password.')
sys.stdout.write('Username: '******'Password: '******'Login: `%s`, Password: `%s`' % (adapter.username, adapter.password))
except exceptions.AdultCheckRequired:
if options.interactive:
print('Please confirm you are an adult in your locale: (y/n)?')
if sys.stdin.readline().strip().lower().startswith('y'):
adapter.is_adult = True
else:
print('Adult check required on non-interactive process. Set is_adult:true in personal.ini or pass -o "is_adult=true" to the command.')
return
if options.update and not options.force:
urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata('numChapters').replace(',',''))
# returns int adjusted for start-end range.
urlchaptercount = adapter.getStoryMetadataOnly().getChapterCount()
if chaptercount == urlchaptercount and not options.metaonly:
print('%s already contains %d chapters.' % (output_filename, chaptercount))
elif chaptercount > urlchaptercount:
print('%s contains %d chapters, more than source: %d.' % (output_filename, chaptercount, urlchaptercount))
elif chaptercount == 0:
print("%s doesn't contain any recognizable chapters, probably from a different source. Not updating." % output_filename)
else:
# update now handled by pre-populating the old
# images and chapters in the adapter rather than
# merging epubs.
(url,
chaptercount,
adapter.oldchapters,
adapter.oldimgs,
adapter.oldcover,
adapter.calibrebookmark,
adapter.logfile,
adapter.oldchaptersmap,
adapter.oldchaptersdata) = (get_update_data(output_filename))[0:9]
print('Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount))
if not options.update and chaptercount == urlchaptercount and adapter.getConfig('do_update_hook'):
adapter.hookForUpdates(chaptercount)
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(adapter.getConfig('pre_process_cmd')).substitute(metadata), shell=True)
write_story(configuration, adapter, 'epub')
else:
# regular download
if options.metaonly:
metadata = adapter.getStoryMetadataOnly().getAllMetadata()
if not options.nometachapters:
metadata['zchapters'] = []
for i, chap in enumerate(adapter.get_chapters()):
metadata['zchapters'].append((i+1,chap))
if not options.metaonly and adapter.getConfig('pre_process_cmd'):
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(adapter.getConfig('pre_process_cmd')).substitute(metadata), shell=True)
output_filename = write_story(configuration, adapter, options.format, options.metaonly)
if options.metaonly:
metadata['output_filename'] = output_filename
if options.jsonmeta:
import json
print(json.dumps(metadata, sort_keys=True,
indent=2, separators=(',', ':')))
else:
pprint.pprint(metadata)
if not options.metaonly and adapter.getConfig('post_process_cmd'):
if adapter.getConfig('post_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(pattern=adapter.getConfig('post_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
metadata['output_filename'] = output_filename
call(string.Template(adapter.getConfig('post_process_cmd')).substitute(metadata), shell=True)
del adapter
except exceptions.InvalidStoryURL as isu:
print(isu)
except exceptions.StoryDoesNotExist as dne:
print(dne)
except exceptions.UnknownSite as us:
print(us)
except exceptions.AccessDenied as ad:
print(ad)
def do_download(arg, options, passed_defaultsini, passed_personalini):
# Attempt to update an existing epub.
chaptercount = None
output_filename = None
if options.unnew:
# remove mark_new_chapters marks
reset_orig_chapters_epub(arg, arg)
return
if options.update:
try:
url, chaptercount = get_dcsource_chaptercount(arg)
if not url:
print 'No story URL found in epub to update.'
return
print 'Updating %s, URL: %s' % (arg, url)
output_filename = arg
except Exception:
# if there's an error reading the update file, maybe it's a URL?
# we'll look for an existing outputfile down below.
url = arg
else:
url = arg
configuration = get_configuration(url, passed_defaultsini,
passed_personalini, options,
chaptercount, output_filename)
try:
adapter = adapters.getAdapter(configuration, url)
## Share pagecache and cookiejar between multiple downloads.
if not hasattr(options, 'pagecache'):
options.pagecache = configuration.get_empty_pagecache()
if not hasattr(options, 'cookiejar'):
options.cookiejar = configuration.get_empty_cookiejar()
configuration.set_pagecache(options.pagecache)
configuration.set_cookiejar(options.cookiejar)
adapter.setChaptersRange(options.begin, options.end)
# check for updating from URL (vs from file)
if options.update and not chaptercount:
try:
writer = writers.getWriter('epub', configuration, adapter)
output_filename = writer.getOutputFileName()
noturl, chaptercount = get_dcsource_chaptercount(
output_filename)
print 'Updating %s, URL: %s' % (output_filename, url)
except Exception:
options.update = False
pass
# Check for include_images without no_image_processing. In absence of PIL, give warning.
if adapter.getConfig('include_images') and not adapter.getConfig(
'no_image_processing'):
try:
from calibre.utils.magick import Image
except ImportError:
try:
## Pillow is a more current fork of PIL library
from PIL import Image
except ImportError:
try:
import Image
except ImportError:
print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
if options.interactive:
if not sys.stdin.readline().strip().lower(
).startswith('y'):
return
else:
# for non-interactive, default the response to yes and continue processing
print 'y'
# three tries, that's enough if both user/pass & is_adult needed,
# or a couple tries of one or the other
for x in range(0, 2):
try:
adapter.getStoryMetadataOnly()
except exceptions.FailedToLogin, f:
if not options.interactive:
print 'Login Failed on non-interactive process. Set username and password in personal.ini.'
return
if f.passwdonly:
print 'Story requires a password.'
else:
print 'Login Failed, Need Username/Password.'
sys.stdout.write('Username: '******'Password: '******'Login: `%s`, Password: `%s`' % (adapter.username, adapter.password))
except exceptions.AdultCheckRequired:
if options.interactive:
print 'Please confirm you are an adult in your locale: (y/n)?'
if sys.stdin.readline().strip().lower().startswith('y'):
adapter.is_adult = True
else:
print 'Adult check required on non-interactive process. Set is_adult:true in personal.ini or pass -o "is_adult=true" to the command.'
return
if not hasattr(options, 'pagecache'):
options.pagecache = adapter.get_empty_pagecache()
options.cookiejar = adapter.get_empty_cookiejar()
adapter.set_pagecache(options.pagecache)
adapter.set_cookiejar(options.cookiejar)
adapter.setChaptersRange(options.begin, options.end)
# check for updating from URL (vs from file)
if options.update and not chaptercount:
try:
writer = writers.getWriter('epub', configuration, adapter)
output_filename = writer.getOutputFileName()
noturl, chaptercount = get_dcsource_chaptercount(
output_filename)
print 'Updating %s, URL: %s' % (output_filename, url)
except Exception:
options.update = False
pass
# Check for include_images without no_image_processing. In absence of PIL, give warning.
if adapter.getConfig('include_images') and not adapter.getConfig(
'no_image_processing'):
try:
from calibre.utils.magick import Image
logging.debug('Using calibre.utils.magick')
except ImportError:
try:
import Image
def do_download(arg,
options,
passed_defaultsini,
passed_personalini):
# Attempt to update an existing epub.
chaptercount = None
output_filename = None
if options.unnew:
# remove mark_new_chapters marks
reset_orig_chapters_epub(arg,arg)
return
if options.update:
try:
url, chaptercount = get_dcsource_chaptercount(arg)
if not url:
print 'No story URL found in epub to update.'
return
print 'Updating %s, URL: %s' % (arg, url)
output_filename = arg
except Exception:
# if there's an error reading the update file, maybe it's a URL?
# we'll look for an existing outputfile down below.
url = arg
else:
url = arg
configuration = get_configuration(url,
passed_defaultsini,
passed_personalini,
options,
chaptercount,
output_filename)
try:
adapter = adapters.getAdapter(configuration, url)
if not hasattr(options,'pagecache'):
options.pagecache = adapter.get_empty_pagecache()
options.cookiejar = adapter.get_empty_cookiejar()
adapter.set_pagecache(options.pagecache)
adapter.set_cookiejar(options.cookiejar)
adapter.setChaptersRange(options.begin, options.end)
# check for updating from URL (vs from file)
if options.update and not chaptercount:
try:
writer = writers.getWriter('epub', configuration, adapter)
output_filename = writer.getOutputFileName()
noturl, chaptercount = get_dcsource_chaptercount(output_filename)
print 'Updating %s, URL: %s' % (output_filename, url)
except Exception:
options.update = False
pass
# Check for include_images without no_image_processing. In absence of PIL, give warning.
if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
try:
from calibre.utils.magick import Image
logging.debug('Using calibre.utils.magick')
except ImportError:
try:
## Pillow is a more current fork of PIL library
from PIL import Image
logging.debug('Using Pillow')
except ImportError:
try:
import Image
logging.debug('Using PIL')
except ImportError:
print "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
if not sys.stdin.readline().strip().lower().startswith('y'):
return
# three tries, that's enough if both user/pass & is_adult needed,
# or a couple tries of one or the other
for x in range(0, 2):
try:
adapter.getStoryMetadataOnly()
except exceptions.FailedToLogin, f:
if f.passwdonly:
print 'Story requires a password.'
else:
print 'Login Failed, Need Username/Password.'
sys.stdout.write('Username: '******'Password: '******'Login: `%s`, Password: `%s`' % (adapter.username, adapter.password))
except exceptions.AdultCheckRequired:
print 'Please confirm you are an adult in your locale: (y/n)?'
if sys.stdin.readline().strip().lower().startswith('y'):
adapter.is_adult = True
if not hasattr(options,'pagecache'):
options.pagecache = adapter.get_empty_pagecache()
options.cookiejar = adapter.get_empty_cookiejar()
adapter.set_pagecache(options.pagecache)
adapter.set_cookiejar(options.cookiejar)
adapter.setChaptersRange(options.begin, options.end)
# check for updating from URL (vs from file)
if options.update and not chaptercount:
try:
writer = writers.getWriter('epub', configuration, adapter)
output_filename = writer.getOutputFileName()
noturl, chaptercount = get_dcsource_chaptercount(output_filename)
print 'Updating %s, URL: %s' % (output_filename, url)
except Exception:
options.update = False
pass
# Check for include_images without no_image_processing. In absence of PIL, give warning.
if adapter.getConfig('include_images') and not adapter.getConfig('no_image_processing'):
try:
from calibre.utils.magick import Image
logging.debug('Using calibre.utils.magick')
except ImportError:
try:
import Image