def post(self): logging.getLogger().setLevel(logging.DEBUG) user = users.get_current_user() if not user: self.redirect(users.create_login_url(self.request.uri)) return format = self.request.get('format') url = self.request.get('url') if not url or url.strip() == "": self.redirect('/') return # Allow chapter range with URL. # like test1.com?sid=5[4-6] or [4,6] url,ch_begin,ch_end = adapters.get_url_chapter_range(url) logging.info("Queuing Download: %s" % url) login = self.request.get('login') password = self.request.get('password') is_adult = self.request.get('is_adult') == "on" email = self.request.get('email') # use existing record if available. Fetched/Created before # the adapter can normalize the URL in case we need to record # an exception. download = getDownloadMeta(url=url,user=user,format=format,new=True) adapter = None try: try: configuration = self.getUserConfig(user,url,format) except exceptions.UnknownSite: self.redirect("/?error=custom&errtext=%s"%urllib.quote("Unsupported site in URL (%s). See 'Support sites' list below."%url,'')) return except Exception as e: self.redirect("/?error=custom&errtext=%s"%urllib.quote("There's an error in your User Configuration: "+unicode(e),'')[:2048]) # limited due to Locatton header length limit. return adapter = adapters.getAdapter(configuration,url) adapter.setChaptersRange(ch_begin,ch_end) logging.info('Created an adaper: %s' % adapter) if login or password: adapter.username=login adapter.password=password adapter.is_adult=is_adult ## This scrapes the metadata, which will be ## duplicated in the queue task, but it ## detects bad URLs, bad login, bad story, etc ## without waiting for the queue. So I think ## it's worth the double up. Could maybe save ## it all in the download object someday. story = adapter.getStoryMetadataOnly() ## Fetch again using normalized story URL. The one ## fetched/created above, if different, will not be saved. download = getDownloadMeta(url=story.getMetadata('storyUrl'), user=user,format=format,new=True) download.title = story.getMetadata('title') download.author = story.getMetadata('author') download.url = story.getMetadata('storyUrl') download.ch_begin = ch_begin download.ch_end = ch_end download.put() taskqueue.add(url='/fdowntask', queue_name="download", params={'id':unicode(download.key()), 'format':format, 'url':download.url, 'login':login, 'password':password, 'user':user.email(), 'email':email, 'is_adult':is_adult}) logging.info("enqueued download key: " + unicode(download.key())) except (exceptions.FailedToLogin,exceptions.AdultCheckRequired), e: download.failure = unicode(e) download.put() logging.info(unicode(e)) is_login= ( isinstance(e, exceptions.FailedToLogin) ) is_passwdonly = is_login and e.passwdonly template_values = dict(nickname = user.nickname(), url = url, format = format, site = adapter.getConfigSection(), fic = download, is_login=is_login, is_passwdonly=is_passwdonly ) # thewriterscoffeeshop.com can do adult check *and* user required. if isinstance(e,exceptions.AdultCheckRequired): template_values['login']=login template_values['password']=password path = os.path.join(os.path.dirname(__file__), 'login.html') self.response.out.write(template.render(path, template_values)) return
def do_download(arg, options, passed_defaultsini, passed_personalini, warn=print, fail=print): # Attempt to update an existing epub. chaptercount = None output_filename = None if options.unnew: # remove mark_new_chapters marks reset_orig_chapters_epub(arg, arg) return if options.update: try: url, chaptercount = get_dcsource_chaptercount(arg) if not url: fail('No story URL found in epub to update.') return print('Updating %s, URL: %s' % (arg, url)) output_filename = arg except Exception: # if there's an error reading the update file, maybe it's a URL? # we'll look for an existing outputfile down below. url = arg else: url = arg configuration = get_configuration(url, passed_defaultsini, passed_personalini, options, chaptercount, output_filename) try: # Allow chapter range with URL. # like test1.com?sid=5[4-6] or [4,6] # Overrides CLI options if present. url, ch_begin, ch_end = adapters.get_url_chapter_range(url) adapter = adapters.getAdapter(configuration, url) # url[begin-end] overrides CLI option if present. if ch_begin or ch_end: adapter.setChaptersRange(ch_begin, ch_end) else: adapter.setChaptersRange(options.begin, options.end) # check for updating from URL (vs from file) update_story = options.update if update_story and not chaptercount: try: writer = writers.getWriter('epub', configuration, adapter) output_filename = writer.getOutputFileName() noturl, chaptercount = get_dcsource_chaptercount( output_filename) print('Updating %s, URL: %s' % (output_filename, url)) except Exception as e: warn( "Failed to read epub for update: (%s) Continuing with update=false" % e) update_story = False # Check for include_images without no_image_processing. In absence of PIL, give warning. if adapter.getConfig('include_images') and not adapter.getConfig( 'no_image_processing'): try: from calibre.utils.magick import Image except ImportError: try: ## Pillow is a more current fork of PIL library from PIL import Image except ImportError: try: import Image except ImportError: print( "You have include_images enabled, but Python Image Library(PIL) isn't found.\nImages will be included full size in original format.\nContinue? (y/n)?" ) if options.interactive: if not sys.stdin.readline().strip().lower( ).startswith('y'): return else: # for non-interactive, default the response to yes and continue processing print('y') # three tries, that's enough if both user/pass & is_adult needed, # or a couple tries of one or the other for x in range(0, 2): try: adapter.getStoryMetadataOnly() except exceptions.FailedToLogin as f: if not options.interactive: print( 'Login Failed on non-interactive process. Set username and password in personal.ini.' ) return if f.passwdonly: print('Story requires a password.') else: print('Login Failed, Need Username/Password.') sys.stdout.write('Username: '******'Password: '******'Login: `%s`, Password: `%s`' % (adapter.username, adapter.password)) except exceptions.AdultCheckRequired: if options.interactive: print( 'Please confirm you are an adult in your locale: (y/n)?' ) if sys.stdin.readline().strip().lower().startswith('y'): adapter.is_adult = True else: print( 'Adult check required on non-interactive process. Set is_adult:true in personal.ini or pass -o "is_adult=true" to the command.' ) return if update_story and not options.force: urlchaptercount = int(adapter.getStoryMetadataOnly().getMetadata( 'numChapters').replace(',', '')) # returns int adjusted for start-end range. urlchaptercount = adapter.getStoryMetadataOnly().getChapterCount() if chaptercount == urlchaptercount and not options.metaonly and not options.updatealways: print('%s already contains %d chapters.' % (output_filename, chaptercount)) elif chaptercount > urlchaptercount: warn('%s contains %d chapters, more than source: %d.' % (output_filename, chaptercount, urlchaptercount)) elif chaptercount == 0: warn( "%s doesn't contain any recognizable chapters, probably from a different source. Not updating." % output_filename) else: # update now handled by pre-populating the old # images and chapters in the adapter rather than # merging epubs. (url, chaptercount, adapter.oldchapters, adapter.oldimgs, adapter.oldcover, adapter.calibrebookmark, adapter.logfile, adapter.oldchaptersmap, adapter.oldchaptersdata) = ( get_update_data(output_filename))[0:9] print('Do update - epub(%d) vs url(%d)' % (chaptercount, urlchaptercount)) if not update_story and chaptercount == urlchaptercount and adapter.getConfig( 'do_update_hook'): adapter.hookForUpdates(chaptercount) if adapter.getConfig('pre_process_safepattern'): metadata = adapter.story.get_filename_safe_metadata( pattern=adapter.getConfig('pre_process_safepattern')) else: metadata = adapter.story.getAllMetadata() call(string.Template( adapter.getConfig('pre_process_cmd')).substitute(metadata), shell=True) output_filename = write_story(configuration, adapter, 'epub', nooutput=options.nooutput) else: if not options.metaonly and adapter.getConfig('pre_process_cmd'): if adapter.getConfig('pre_process_safepattern'): metadata = adapter.story.get_filename_safe_metadata( pattern=adapter.getConfig('pre_process_safepattern')) else: metadata = adapter.story.getAllMetadata() call(string.Template( adapter.getConfig('pre_process_cmd')).substitute(metadata), shell=True) output_filename = write_story(configuration, adapter, options.format, metaonly=options.metaonly, nooutput=options.nooutput) if options.metaonly and not options.jsonmeta: metadata = adapter.getStoryMetadataOnly().getAllMetadata() metadata['output_filename'] = output_filename if not options.nometachapters: metadata['zchapters'] = [] for i, chap in enumerate(adapter.get_chapters()): metadata['zchapters'].append((i + 1, chap)) else: # If no chapters, also suppress output_css so # metadata is shorter. del metadata['output_css'] pprint.pprint(metadata) if not options.metaonly and adapter.getConfig('post_process_cmd'): if adapter.getConfig('post_process_safepattern'): metadata = adapter.story.get_filename_safe_metadata( pattern=adapter.getConfig('post_process_safepattern')) else: metadata = adapter.story.getAllMetadata() metadata['output_filename'] = output_filename call(string.Template( adapter.getConfig('post_process_cmd')).substitute(metadata), shell=True) if options.jsonmeta or options.jsonmetafile: metadata = adapter.getStoryMetadataOnly().getAllMetadata() metadata['output_filename'] = output_filename if not options.nometachapters: metadata['zchapters'] = [] for i, chap in enumerate(adapter.get_chapters()): metadata['zchapters'].append((i + 1, chap)) import json if options.jsonmeta: print( json.dumps(metadata, sort_keys=True, indent=2, separators=(',', ':'))) if options.jsonmetafile: with open(output_filename + ".json", "w") as jsonfile: json.dump(metadata, jsonfile, sort_keys=True, indent=2, separators=(',', ':')) if adapter.story.chapter_error_count > 0: warn( "===================\n!!!! %s chapters errored downloading %s !!!!\n===================" % (adapter.story.chapter_error_count, url)) del adapter except exceptions.InvalidStoryURL as isu: fail(isu) except exceptions.StoryDoesNotExist as dne: fail(dne) except exceptions.UnknownSite as us: fail(us) except exceptions.AccessDenied as ad: fail(ad)
def post(self): logging.getLogger().setLevel(logging.DEBUG) user = users.get_current_user() if not user: self.redirect(users.create_login_url(self.request.uri)) return format = self.request.get('format') url = self.request.get('url') if not url or url.strip() == "": self.redirect('/') return # Allow chapter range with URL. # like test1.com?sid=5[4-6] or [4,6] url, ch_begin, ch_end = adapters.get_url_chapter_range(url) logging.info("Queuing Download: %s" % url) login = self.request.get('login') password = self.request.get('password') is_adult = self.request.get('is_adult') == "on" email = self.request.get('email') # use existing record if available. Fetched/Created before # the adapter can normalize the URL in case we need to record # an exception. download = getDownloadMeta(url=url, user=user, format=format, new=True) adapter = None try: try: configuration = self.getUserConfig(user, url, format) except exceptions.UnknownSite: self.redirect("/?error=custom&errtext=%s" % urllib.quote( "Unsupported site in URL (%s). See 'Support sites' list below." % url, '')) return except Exception as e: self.redirect("/?error=custom&errtext=%s" % urllib.quote( "There's an error in your User Configuration: " + unicode(e), '')[:2048]) # limited due to Locatton header length limit. return adapter = adapters.getAdapter(configuration, url) adapter.setChaptersRange(ch_begin, ch_end) logging.info('Created an adaper: %s' % adapter) if login or password: adapter.username = login adapter.password = password adapter.is_adult = is_adult ## This scrapes the metadata, which will be ## duplicated in the queue task, but it ## detects bad URLs, bad login, bad story, etc ## without waiting for the queue. So I think ## it's worth the double up. Could maybe save ## it all in the download object someday. story = adapter.getStoryMetadataOnly() ## Fetch again using normalized story URL. The one ## fetched/created above, if different, will not be saved. download = getDownloadMeta(url=story.getMetadata('storyUrl'), user=user, format=format, new=True) download.title = story.getMetadata('title') download.author = story.getMetadata('author') download.url = story.getMetadata('storyUrl') download.ch_begin = ch_begin download.ch_end = ch_end download.put() taskqueue.add(url='/fdowntask', queue_name="download", params={ 'id': unicode(download.key()), 'format': format, 'url': download.url, 'login': login, 'password': password, 'user': user.email(), 'email': email, 'is_adult': is_adult }) logging.info("enqueued download key: " + unicode(download.key())) except (exceptions.FailedToLogin, exceptions.AdultCheckRequired), e: download.failure = unicode(e) download.put() logging.info(unicode(e)) is_login = (isinstance(e, exceptions.FailedToLogin)) is_passwdonly = is_login and e.passwdonly template_values = dict(nickname=user.nickname(), url=url, format=format, site=adapter.getConfigSection(), fic=download, is_login=is_login, is_passwdonly=is_passwdonly) # thewriterscoffeeshop.com can do adult check *and* user required. if isinstance(e, exceptions.AdultCheckRequired): template_values['login'] = login template_values['password'] = password path = os.path.join(os.path.dirname(__file__), 'login.html') self.response.out.write(template.render(path, template_values)) return