def post(self):
logging.getLogger().setLevel(logging.DEBUG)
user = users.get_current_user()
if not user:
self.redirect(users.create_login_url(self.request.uri))
return
format = self.request.get('format')
url = self.request.get('url')
if not url or url.strip() == "":
self.redirect('/')
return
# Allow chapter range with URL.
# like test1.com?sid=5[4-6] or [4,6]
url,ch_begin,ch_end = adapters.get_url_chapter_range(url)
logging.info("Queuing Download: %s" % url)
login = self.request.get('login')
password = self.request.get('password')
is_adult = self.request.get('is_adult') == "on"
email = self.request.get('email')
# use existing record if available. Fetched/Created before
# the adapter can normalize the URL in case we need to record
# an exception.
download = getDownloadMeta(url=url,user=user,format=format,new=True)
adapter = None
try:
try:
configuration = self.getUserConfig(user,url,format)
except exceptions.UnknownSite:
self.redirect("/?error=custom&errtext=%s"%urllib.quote("Unsupported site in URL (%s). See 'Support sites' list below."%url,''))
return
except Exception as e:
self.redirect("/?error=custom&errtext=%s"%urllib.quote("There's an error in your User Configuration: "+unicode(e),'')[:2048]) # limited due to Locatton header length limit.
return
adapter = adapters.getAdapter(configuration,url)
adapter.setChaptersRange(ch_begin,ch_end)
logging.info('Created an adaper: %s' % adapter)
if login or password:
adapter.username=login
adapter.password=password
adapter.is_adult=is_adult
## This scrapes the metadata, which will be
## duplicated in the queue task, but it
## detects bad URLs, bad login, bad story, etc
## without waiting for the queue. So I think
## it's worth the double up. Could maybe save
## it all in the download object someday.
story = adapter.getStoryMetadataOnly()
## Fetch again using normalized story URL. The one
## fetched/created above, if different, will not be saved.
download = getDownloadMeta(url=story.getMetadata('storyUrl'),
user=user,format=format,new=True)
download.title = story.getMetadata('title')
download.author = story.getMetadata('author')
download.url = story.getMetadata('storyUrl')
download.ch_begin = ch_begin
download.ch_end = ch_end
download.put()
taskqueue.add(url='/fdowntask',
queue_name="download",
params={'id':unicode(download.key()),
'format':format,
'url':download.url,
'login':login,
'password':password,
'user':user.email(),
'email':email,
'is_adult':is_adult})
logging.info("enqueued download key: " + unicode(download.key()))
except (exceptions.FailedToLogin,exceptions.AdultCheckRequired), e:
download.failure = unicode(e)
download.put()
logging.info(unicode(e))
is_login= ( isinstance(e, exceptions.FailedToLogin) )
is_passwdonly = is_login and e.passwdonly
template_values = dict(nickname = user.nickname(),
url = url,
format = format,
site = adapter.getConfigSection(),
fic = download,
is_login=is_login,
is_passwdonly=is_passwdonly
)
# thewriterscoffeeshop.com can do adult check *and* user required.
if isinstance(e,exceptions.AdultCheckRequired):
template_values['login']=login
template_values['password']=password
path = os.path.join(os.path.dirname(__file__), 'login.html')
self.response.out.write(template.render(path, template_values))
return
def do_download(arg,
options,
passed_defaultsini,
passed_personalini,
warn=print,
fail=print):
# Attempt to update an existing epub.
chaptercount = None
output_filename = None
if options.unnew:
# remove mark_new_chapters marks
reset_orig_chapters_epub(arg, arg)
return
if options.update:
try:
url, chaptercount = get_dcsource_chaptercount(arg)
if not url:
fail('No story URL found in epub to update.')
return
print('Updating %s, URL: %s' % (arg, url))
output_filename = arg
except Exception:
# if there's an error reading the update file, maybe it's a URL?
# we'll look for an existing outputfile down below.
url = arg
else:
url = arg
configuration = get_configuration(url, passed_defaultsini,
passed_personalini, options,
chaptercount, output_filename)
try:
# Allow chapter range with URL.
# like test1.com?sid=5[4-6] or [4,6]
# Overrides CLI options if present.
url, ch_begin, ch_end = adapters.get_url_chapter_range(url)
adapter = adapters.getAdapter(configuration, url)
# url[begin-end] overrides CLI option if present.
if ch_begin or ch_end:
adapter.setChaptersRange(ch_begin, ch_end)
else:
adapter.setChaptersRange(options.begin, options.end)
# check for updating from URL (vs from file)
update_story = options.update
if update_story and not chaptercount:
try:
writer = writers.getWriter('epub', configuration, adapter)
output_filename = writer.getOutputFileName()
noturl, chaptercount = get_dcsource_chaptercount(
output_filename)
print('Updating %s, URL: %s' % (output_filename, url))
except Exception as e:
warn(
"Failed to read epub for update: (%s) Continuing with update=false"
% e)
update_story = False
# Check for include_images without no_image_processing. In absence of PIL, give warning.
if adapter.getConfig('include_images') and not adapter.getConfig(
'no_image_processing'):
try:
from calibre.utils.magick import Image
except ImportError:
try:
## Pillow is a more current fork of PIL library
from PIL import Image
except ImportError:
try:
import Image
except ImportError:
print(
"You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?"
)
if options.interactive:
if not sys.stdin.readline().strip().lower(
).startswith('y'):
return
else:
# for non-interactive, default the response to yes and continue processing
print('y')
# three tries, that's enough if both user/pass & is_adult needed,
# or a couple tries of one or the other
for x in range(0, 2):
try:
adapter.getStoryMetadataOnly()
except exceptions.FailedToLogin as f:
if not options.interactive:
print(
'Login Failed on non-interactive process. Set username and password in personal.ini.'
)
return
if f.passwdonly:
print('Story requires a password.')
else:
print('Login Failed, Need Username/Password.')
sys.stdout.write('Username: '******'Password: '******'Login: `%s`, Password: `%s`' % (adapter.username, adapter.password))
except exceptions.AdultCheckRequired:
if options.interactive:
print(
'Please confirm you are an adult in your locale: (y/n)?'
)
if sys.stdin.readline().strip().lower().startswith('y'):
adapter.is_adult = True
else:
print(
'Adult check required on non-interactive process. Set is_adult:true in personal.ini or pass -o "is_adult=true" to the command.'
)
return
if update_story and not options.force:
urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata(
'numChapters').replace(',', ''))
# returns int adjusted for start-end range.
urlchaptercount = adapter.getStoryMetadataOnly().getChapterCount()
if chaptercount == urlchaptercount and not options.metaonly and not options.updatealways:
print('%s already contains %d chapters.' %
(output_filename, chaptercount))
elif chaptercount > urlchaptercount:
warn('%s contains %d chapters, more than source: %d.' %
(output_filename, chaptercount, urlchaptercount))
elif chaptercount == 0:
warn(
"%s doesn't contain any recognizable chapters, probably from a different source. Not updating."
% output_filename)
else:
# update now handled by pre-populating the old
# images and chapters in the adapter rather than
# merging epubs.
(url, chaptercount, adapter.oldchapters, adapter.oldimgs,
adapter.oldcover, adapter.calibrebookmark, adapter.logfile,
adapter.oldchaptersmap, adapter.oldchaptersdata) = (
get_update_data(output_filename))[0:9]
print('Do update - epub(%d) vs url(%d)' %
(chaptercount, urlchaptercount))
if not update_story and chaptercount == urlchaptercount and adapter.getConfig(
'do_update_hook'):
adapter.hookForUpdates(chaptercount)
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(
pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(
adapter.getConfig('pre_process_cmd')).substitute(metadata),
shell=True)
output_filename = write_story(configuration,
adapter,
'epub',
nooutput=options.nooutput)
else:
if not options.metaonly and adapter.getConfig('pre_process_cmd'):
if adapter.getConfig('pre_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(
pattern=adapter.getConfig('pre_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
call(string.Template(
adapter.getConfig('pre_process_cmd')).substitute(metadata),
shell=True)
output_filename = write_story(configuration,
adapter,
options.format,
metaonly=options.metaonly,
nooutput=options.nooutput)
if options.metaonly and not options.jsonmeta:
metadata = adapter.getStoryMetadataOnly().getAllMetadata()
metadata['output_filename'] = output_filename
if not options.nometachapters:
metadata['zchapters'] = []
for i, chap in enumerate(adapter.get_chapters()):
metadata['zchapters'].append((i + 1, chap))
else:
# If no chapters, also suppress output_css so
# metadata is shorter.
del metadata['output_css']
pprint.pprint(metadata)
if not options.metaonly and adapter.getConfig('post_process_cmd'):
if adapter.getConfig('post_process_safepattern'):
metadata = adapter.story.get_filename_safe_metadata(
pattern=adapter.getConfig('post_process_safepattern'))
else:
metadata = adapter.story.getAllMetadata()
metadata['output_filename'] = output_filename
call(string.Template(
adapter.getConfig('post_process_cmd')).substitute(metadata),
shell=True)
if options.jsonmeta or options.jsonmetafile:
metadata = adapter.getStoryMetadataOnly().getAllMetadata()
metadata['output_filename'] = output_filename
if not options.nometachapters:
metadata['zchapters'] = []
for i, chap in enumerate(adapter.get_chapters()):
metadata['zchapters'].append((i + 1, chap))
import json
if options.jsonmeta:
print(
json.dumps(metadata,
sort_keys=True,
indent=2,
separators=(',', ':')))
if options.jsonmetafile:
with open(output_filename + ".json", "w") as jsonfile:
json.dump(metadata,
jsonfile,
sort_keys=True,
indent=2,
separators=(',', ':'))
if adapter.story.chapter_error_count > 0:
warn(
"===================\n!!!! %s chapters errored downloading %s !!!!\n==================="
% (adapter.story.chapter_error_count, url))
del adapter
except exceptions.InvalidStoryURL as isu:
fail(isu)
except exceptions.StoryDoesNotExist as dne:
fail(dne)
except exceptions.UnknownSite as us:
fail(us)
except exceptions.AccessDenied as ad:
fail(ad)
def post(self):
logging.getLogger().setLevel(logging.DEBUG)
user = users.get_current_user()
if not user:
self.redirect(users.create_login_url(self.request.uri))
return
format = self.request.get('format')
url = self.request.get('url')
if not url or url.strip() == "":
self.redirect('/')
return
# Allow chapter range with URL.
# like test1.com?sid=5[4-6] or [4,6]
url, ch_begin, ch_end = adapters.get_url_chapter_range(url)
logging.info("Queuing Download: %s" % url)
login = self.request.get('login')
password = self.request.get('password')
is_adult = self.request.get('is_adult') == "on"
email = self.request.get('email')
# use existing record if available. Fetched/Created before
# the adapter can normalize the URL in case we need to record
# an exception.
download = getDownloadMeta(url=url, user=user, format=format, new=True)
adapter = None
try:
try:
configuration = self.getUserConfig(user, url, format)
except exceptions.UnknownSite:
self.redirect("/?error=custom&errtext=%s" % urllib.quote(
"Unsupported site in URL (%s). See 'Support sites' list below."
% url, ''))
return
except Exception as e:
self.redirect("/?error=custom&errtext=%s" % urllib.quote(
"There's an error in your User Configuration: " +
unicode(e),
'')[:2048]) # limited due to Locatton header length limit.
return
adapter = adapters.getAdapter(configuration, url)
adapter.setChaptersRange(ch_begin, ch_end)
logging.info('Created an adaper: %s' % adapter)
if login or password:
adapter.username = login
adapter.password = password
adapter.is_adult = is_adult
## This scrapes the metadata, which will be
## duplicated in the queue task, but it
## detects bad URLs, bad login, bad story, etc
## without waiting for the queue. So I think
## it's worth the double up. Could maybe save
## it all in the download object someday.
story = adapter.getStoryMetadataOnly()
## Fetch again using normalized story URL. The one
## fetched/created above, if different, will not be saved.
download = getDownloadMeta(url=story.getMetadata('storyUrl'),
user=user,
format=format,
new=True)
download.title = story.getMetadata('title')
download.author = story.getMetadata('author')
download.url = story.getMetadata('storyUrl')
download.ch_begin = ch_begin
download.ch_end = ch_end
download.put()
taskqueue.add(url='/fdowntask',
queue_name="download",
params={
'id': unicode(download.key()),
'format': format,
'url': download.url,
'login': login,
'password': password,
'user': user.email(),
'email': email,
'is_adult': is_adult
})
logging.info("enqueued download key: " + unicode(download.key()))
except (exceptions.FailedToLogin, exceptions.AdultCheckRequired), e:
download.failure = unicode(e)
download.put()
logging.info(unicode(e))
is_login = (isinstance(e, exceptions.FailedToLogin))
is_passwdonly = is_login and e.passwdonly
template_values = dict(nickname=user.nickname(),
url=url,
format=format,
site=adapter.getConfigSection(),
fic=download,
is_login=is_login,
is_passwdonly=is_passwdonly)
# thewriterscoffeeshop.com can do adult check *and* user required.
if isinstance(e, exceptions.AdultCheckRequired):
template_values['login'] = login
template_values['password'] = password
path = os.path.join(os.path.dirname(__file__), 'login.html')
self.response.out.write(template.render(path, template_values))
return