def load_report_by_sources(self,
sources,
limit=None,
offset=None,
keywords=None):
items = self.find_items_by_values(sources, "source", limit, offset,
keywords)
items = time_localizer("published", items)
items = data_updater("founds", "founds", str2list, True, items)
items = data_updater("id", "id", id2hashid, True, items)
return items
def load_report_daily(self, page=None, limit=None, keywords=None, sources=None):
offset = (page - 1) * limit
start = arrow.now(self.tzinfo).shift(days=-1).replace(hour=8).floor('hour').datetime
end = arrow.now(self.tzinfo).replace(hour=7).ceil('hour').datetime
if sources:
items = self.find_items_by_values_and_datetime_between(
sources, "source", "published", start, end, limit, offset, keywords)
else:
items = self.find_items_by_datetime_between(
"published", start, end, limit, offset, keywords)
items = data_updater("founds", "founds", str2list, True, items)
items = data_updater("id", "id", id2hashid, True, items)
items = data_updater("published_humanize", "published", local_humanize, True, items)
return items
def _data_produce(self, items):
items = data_cleaner("message", items)
items = data_cleaner("story", items)
items = data_inserter(self.include_text, "keyword", items)
items = dict_renamer("fbid", "id", items)
items = dict_renamer("link", "permalink_url", items)
items = dict_renamer("published", "created_time", items)
items = data_hasher("hash", ["fbid", "published"], items)
items = data_updater("from_id", "from", lambda x: x["id"], True, items)
items = data_updater("from_name", "from", lambda x: x["name"], True,
items)
items = dict_blocker(["from"], items)
items = data_inserter(self.fbid, "source", items)
return items
async def as_load_report_by_page(self, page=1, limit=10, keywords=None, sources=None):
offset = (page - 1) * limit
if sources:
items = await self.as_load_report_by_sources(sources, limit, offset, keywords)
else:
items = await self.as_load_report_all(limit, offset, keywords)
items = data_updater("published_humanize", "published", local_humanize, True, items)
return items
async def as_request_report(request, time='any', hightlight=True, count=False):
data = {}
page = 1
limit = 10
data['page'] = sint(request.args.get('page', page), page)
data['limit'] = sint(request.args.get('limit', limit), limit)
data['category'] = request.args.get('cat', None)
if data['category']:
data['category_zh_tw'] = load_lang('zh_tw', data['category'])
keyword = request.args.get('keyword', None)
data['keyword'] = keyword
data['_keyword'] = keyword
data['items'] = await as_report(time, data['page'], data['limit'],
data['_keyword'], data['category'])
data['parent'] = request.args.get('parent', None)
if data['parent'] and data['_keyword']:
data['_keyword'] = " ".join({data['_keyword'], data['parent']})
if count:
data['count'] = await as_report_count(time, data['_keyword'],
data['category'])
data['parent_count'] = await as_report_count(time, data['parent'],
data['category'])
if hightlight and data['keyword']:
data['items'] = data_updater(
"title", "title",
partial(hightlight_keywords, keywords=data['keyword']), True,
data['items'])
data['items'] = data_updater(
"summary", "summary",
partial(hightlight_keywords, keywords=data['keyword']), True,
data['items'])
if data['keyword']:
(data['_keyword'], _) = reload_keyword(data['_keyword'])
return data
def wrapper(*args, **kargs):
nonlocal keys
args = list(args)
items = args[1]
if not isiterable(keys):
keys = clist(keys)
if not isiterable(items):
items = clist(items)
for key in keys:
if all(key in item for item in items):
items = data_updater(key, key, list2str, True, items)
args[1] = items
result = func(*args, **kargs)
return result
def wrapper(*args, **kargs):
import types
nonlocal keys
args = list(args)
items = args[1]
if not isiterable(keys):
keys = clist(keys)
if not isiterable(items):
items = clist(items)
for key in keys:
items = data_updater(key, key, datetime_encapsulator, True,
items)
args[1] = items
result = func(*args, **kargs)
return result
def _data_prepare(self, items):
self.count = len(items)
items = items[:self.connections]
keys = ['title', 'published', 'link', 'summary', 'updated', 'full_text']
items = dict_filter(keys, items)
items = data_updater("summary", "content", lambda content: content[0][
'value'], all("content" in item for item in items), items)
items = time_corrector("published", items)
items = time_corrector("updated", items)
items = link_corrector("link", items)
if not self.full_text:
if 'supplements' == self.source:
items = data_updater("source", "link", detect_news_source, True, items)
else:
items = data_updater("source", "link", detect_news_source, self.url, items)
else:
items = data_updater("source", "link", detect_news_source, True, items)
items = data_remover("any", "source", items)
remote_items = data_kv_updater_all_load("link", partial(
fetch_news_all, source=self.source), self.full_text, items)
items = data_kv_updater_all_by_remote_items(
remote_items, "summary", "summary", self.full_text, items)
items = data_kv_updater_all_by_remote_items(
remote_items, "published", "published", self.full_text, items)
items = data_kv_updater_all_by_remote_items(
remote_items, "source", "source", self.full_text, items)
items = data_cleaner("summary", items)
items = data_cleaner("title", items)
return items
def load_report_all(self, limit=None, offset=None, keyword=None):
items = self.find_all(limit, offset, keyword)
items = time_localizer("published", items)
items = data_updater("founds", "founds", str2list, True, items)
items = data_updater("id", "id", id2hashid, True, items)
return items
def test_sqlite_datetime_compatibility(self):
items = data_updater("published", "published", datetime_encapsulator, True, self.items)
self.assertEqual(self.items, items)