def check_integrity(self, date_range):
"""Check data integrity, make sure we won't miss a day
:param date_range: int number or 'all'.
the int number means the range of days to check;
string 'all' means check data from start 20130519.
"""
click.echo("Checking date integrity...")
if isinstance(date_range, int):
date_in_db = [
news.date for news in
(Zhihudaily.select(Zhihudaily.date)
.order_by(Zhihudaily.date.desc())
.limit(date_range))
]
delta = date_range
elif date_range == 'all':
date_in_db = [
news.date for news in Zhihudaily.select(Zhihudaily.date)
]
delta = (self.today - self.birthday).days + 1
else:
raise TypeError("Bad parameter date_range, "
"should be an integer or string value 'all'.")
date_in_real = [
(self.today - datetime.timedelta(i)).strftime("%Y%m%d")
for i in range(delta)
]
missed_date = set(date_in_real) - set(date_in_db)
for date in missed_date:
click.echo(" fetching {0}...".format(date))
self._save_to_database(str(date))
click.echo("Check data integrity: done.")
def show_titles(date):
"""Get titles via AJAX."""
try:
news = Zhihudaily.get(Zhihudaily.date == date)
json_news = json.loads(news.json_news)
except Zhihudaily.DoesNotExist:
json_news = []
return jsonify(news=json_news)
def index():
"""The index page, for 文字 UI."""
news = Zhihudaily.select().order_by(Zhihudaily.date.desc()).first()
day = Date(news.date)
return render_template("index.html",
lists=json.loads(news.json_news),
display_date=news.display_date,
day_before=day.day_before)
def with_image():
"""The page for 图片 UI."""
news = Zhihudaily.select().order_by(Zhihudaily.date.desc()).first()
day = Date(news.date)
return render_template('with_image.html',
lists=json.loads(news.json_news),
display_date=news.display_date,
day_before=day.day_before)
def pages(page):
"""The page the 分页 UI."""
db_news_list = (Zhihudaily.select().order_by(
Zhihudaily.date.desc()).paginate(page, 4))
records = [{
"news": json.loads(news.json_news),
"display_date": news.display_date
} for news in db_news_list]
pagination = Pagination(page=page,
total=Zhihudaily.select().count(),
per_page=4,
inner_window=7,
outer_window=3,
css_framework='bootstrap3')
return render_template('pages.html',
records=records,
pagination=pagination)
def _save_to_database(self, given_date):
"""Save news on the specified date to the database.
:param given_date: string type, e.g. '20151106'.
"""
response = self._get_news(given_date)
if response is None:
return
if Zhihudaily.select().where(
Zhihudaily.date == given_date).exists():
zhihudaily = Zhihudaily.get(Zhihudaily.date == given_date)
zhihudaily.json_news = json.dumps(handle_image(response['news']))
zhihudaily.save()
else:
Zhihudaily.create(
date=response['date'],
display_date=response['display_date'],
json_news=json.dumps(handle_image(response['news']))
)
def generate_feed():
"""Code snippet from https://flask.pocoo.org/snippets/10/"""
day = Date()
feed = AtomFeed('Zhihudaily', feed_url=request.url, url=request.url_root)
news = Zhihudaily.get(Zhihudaily.date == int(day.today))
articles = json.loads(news.json_news)
for article in articles:
if redis_server.get(article['url']):
body = redis_server.get(article['url']).decode('utf-8')
else:
body = Crawler().send_request(article['url']).json()['body']
redis_server.setex(article['url'], (60 * 60 * 24), body)
feed.add(article['title'],
body,
content_type='html',
author='zhihudaily',
url=urljoin(request.url_root, article['url']),
updated=day.now)
return feed.get_response()
def before(date):
"""For 文字 UI and 图片 UI, before today."""
day = Date(date)
if day.today <= date:
if request.args.get('image', 'False') == 'True':
return redirect(url_for('image_ui.with_image'))
else:
return redirect(url_for('text_ui.index'))
news = Zhihudaily.get(Zhihudaily.date == date)
template_name = {
'False': 'index.html', 'True': 'with_image.html'
}[request.args.get('image', 'False')]
return render_template(template_name,
lists=json.loads(news.json_news),
display_date=news.display_date,
day_before=day.day_before,
day_after=day.day_after)