def viewsaddsourceall():
vs = get_collection('views_sources')
sources = get_collection('sources')
fields = request.get_json(force=True)
viewName = fields.get('viewName', None)
is_success_pool = {}
for n in [_.get('machine_name') for _ in sources.find().limit(1000)]:
condition = {'viewName': viewName, 'sourceName': n}
to_set = condition.copy()
is_success = vs.update_one(condition, {'$set': to_set}, True)
is_success_pool[n] = is_success.acknowledged
return jsonify({'is_success_pool': is_success_pool})
def viewslist():
views = get_collection('views')
vs = get_collection('views_sources')
data = []
data.extend([
clean_id(i)
for i in views.find().sort('_id', 1).skip(int(0)).limit(int(100))
])
for _ in data:
count = vs.find({'viewName': _.get('name')}).count()
_['count'] = count
ret = {'data': data, 'total': views.find().count()}
return jsonify(ret)
def weibo_news():
items = get_collection('weibo_items')
limit = request.args.get('limit', 1)
skip = request.args.get('skip', 0)
skip = skip if skip != 0 else request.args.get(
'offset', 0) # allow another namespace 'offset'
keyword = request.args.get('keyword', None)
is_keyword_full = request.args.get('is_keyword_full', 0)
startwith = request.args.get('startwith', None)
sql_filter = {}
if keyword:
if int(json.loads(
str(is_keyword_full))): # is_keyword_full maybe "false" or "0"
sql_filter.update({"title": keyword})
else:
sql_filter.update({"title": {'$regex': keyword, '$options': 'i'}})
if startwith:
sql_filter.update({'_id': {'$lte': ObjectId(startwith)}})
data = []
data.extend([
clean_id(i) for i in items.find(sql_filter).sort([(
'_id', -1)]).skip(int(skip)).limit(int(limit))
])
ret = {'data': data, 'total': items.find().count()}
return jsonify(ret)
def viewsdelete():
views = get_collection('views')
fields = request.get_json(force=True)
viewName = fields.get('name', None)
is_success = views.delete_many({'name': viewName})
return jsonify({'is_success': is_success.acknowledged})
def sources():
sources = get_collection('sources')
limit = request.args.get('limit', 50)
skip = request.args.get('skip', 0)
skip = skip if skip != 0 else request.args.get(
'offset', 0) # allow another namespace 'offset'
keyword = request.args.get('keyword', None)
start_urls_keyword = request.args.get('start_urls_keyword', None)
comment_keyword = request.args.get('comment_keyword', None)
startwith = request.args.get('startwith', None)
coop = request.args.get('coop', None)
sql_filter = {}
if keyword:
sql_filter.update({"title": {'$regex': keyword, '$options': 'i'}})
if start_urls_keyword:
sql_filter.update(
{"start_urls": {
'$regex': start_urls_keyword,
'$options': 'i'
}})
if comment_keyword:
sql_filter.update(
{"comments": {
'$regex': comment_keyword,
'$options': 'i'
}})
if startwith:
sql_filter.update({'_id': {'$lte': ObjectId(startwith)}})
if coop:
sql_filter.update({'coops': coop})
data = []
data.extend([
clean_id(i)
for i in sources.find(sql_filter).sort('machine_name', 1).skip(
int(skip)).limit(int(limit))
])
for d in data:
if d.get('latestArticleCreated'):
last_time = Arrow.fromtimestamp(
d['latestArticleCreated']).to('Asia/Shanghai')
d['latestArticleCreatedHuman'] = u'最后时间:{0}'.format(
last_time.humanize(locale='zh_cn'))
if (Arrow.utcnow() - last_time).days > 1:
d['latestArticleCreatedHuman'] = u'<span class="text-danger">{}</span>'.format(
d['latestArticleCreatedHuman'])
ret = {'data': data, 'total': sources.find().count()}
return jsonify(ret)
def viewsadd():
views = get_collection('views')
fieldsFromReq = request.get_json(force=True)
fieldsFilted = filterFields(fieldsFromReq, ['name'])
if not fieldsFilted:
return jsonify({'is_success': True})
fieldsForInsert = fieldsFilted.copy()
fieldsForInsert.update({"first_created": time.time()})
is_success = views.insert_one(fieldsForInsert)
return jsonify({'is_success': is_success.acknowledged})
def sourcedefaultedit():
sources = get_collection('sourceDefaultConfigs')
update = request.get_json(force=True)
fieldsFilted = filterFields(update, ['default_proxy', 'default_delay'])
if not fieldsFilted:
return jsonify({'is_success': True})
fieldsToUpdate = fieldsFilted.copy()
fieldsToUpdate.update({"last_modified": time.time()})
result = sources.find_one_and_update({}, {'$set': fieldsToUpdate})
is_success = True if result.get('_id') else False
return jsonify({'is_success': is_success})
def news2():
items = get_collection('items')
limit = request.args.get('limit', 1)
skip = request.args.get('skip', 0)
source = request.args.get('source', None)
sql_filter = {}
if source:
sql_filter.update({'name': source})
data = []
data.extend([
i for i in items.find(sql_filter, {
'_id': 0
}).sort([('ctime', -1)]).skip(int(skip)).limit(int(limit))
])
ret = {'results': data, 'code': 0}
return jsonp(ret)
def viewsgetsource():
vs = get_collection('views_sources')
fields = request.get_json(force=True)
viewName = fields.get('viewName', None)
# sourceName = fields.get('sourceName', None)
data = []
total = 0
if viewName:
data.extend([clean_id(i) for i in vs.find({'viewName': viewName})])
total = vs.find({'viewName': viewName}).count()
ret = {'data': data, 'total': total}
return jsonify(ret)
def viewsaddsource():
vs = get_collection('views_sources')
fields = request.get_json(force=True)
viewName = fields.get('viewName', None)
sourceName = fields.get('sourceName', None)
importance = fields.get('importance', None)
condition = {'viewName': viewName, 'sourceName': sourceName}
to_set = condition.copy()
if importance != None:
to_set['importance'] = importance
is_success = vs.update_one(condition, {'$set': to_set}, True)
return jsonify({'is_success': is_success.acknowledged})
def news_relate_article():
news = get_collection('items')
fields = request.get_json(force=True)
newsId = fields.get('newsId', None)
baoArticleId = fields.get('baoArticleId', None)
publishedAt = fields.get('publishedAt', None)
fieldsToUpdate = {
'related_bao_article_id': baoArticleId,
'related_bao_article_published': publishedAt,
}
fieldsToUpdate.update({"last_modified": time.time()})
is_success = news.update_one(
{'_id': ObjectId(newsId)},
{'$set': fieldsToUpdate},
)
return jsonify({'is_success': is_success.acknowledged})
def sourceadd():
sources = get_collection('sources')
fieldsFromReq = request.get_json(force=True)
fieldsFilted = filterFields(fieldsFromReq, [
'title', 'proxy_type', 'disabled', 'delay_sec', 'machine_name',
'comments', 'tags', 'star', 'start_urls', 'is_backend_render',
'content_selector', 'removing_style_and_script',
'use_link_content_as_detail_title', 'url_links_selector',
'url_area_selector', 'url_patterns', 'headers', 'coops'
])
if not fieldsFilted:
return jsonify({'json_is_success': True})
fieldsForInsert = fieldsFilted.copy()
fieldsForInsert.update({"first_created": time.time()})
is_success = sources.insert_one(fieldsForInsert)
return jsonify({'inserted_is_success': is_success.acknowledged})
def sourceedit(sourceId):
sources = get_collection('sources')
update = request.get_json(force=True)
fieldsFilted = filterFields(update, [
'title', 'proxy_type', 'disabled', 'delay_sec', 'machine_name',
'comments', 'tags', 'star', 'start_urls', 'is_backend_render',
'content_selector', 'removing_style_and_script',
'use_link_content_as_detail_title', 'url_links_selector',
'url_area_selector', 'url_patterns', 'headers', 'coops'
])
if not fieldsFilted:
return jsonify({'is_success': True})
fieldsToUpdate = fieldsFilted.copy()
fieldsToUpdate.update({"last_modified": time.time()})
is_success = sources.update_one(
{'_id': ObjectId(sourceId)},
{'$set': fieldsToUpdate},
)
return jsonify({'is_success': is_success.acknowledged})
def names():
items = get_collection('items')
if request.method == 'GET':
name_list = items.distinct('name')
current = []
if os.path.exists('filter.json'):
with open('filter.json', 'r') as f:
current = json.load(f)
msgs = {}
for name in name_list:
item = [
i for i in items.find({
'name': name
}, {
'_id': 0
}).sort([('ctime', -1)]).limit(1)
]
if item:
item = item[0]
last_time = Arrow.fromtimestamp(
item['ctime']).to('Asia/Shanghai')
msgs[name] = u'最后时间:{0}'.format(
last_time.humanize(locale='zh_cn'))
if (Arrow.utcnow() - last_time).days > 1:
msgs[name] = u'<span class="text-danger">{}</span>'.format(
msgs[name])
return jsonp({'all': name_list, 'current': current, 'msg': msgs})
elif request.method == 'POST':
code = 500
if request.json:
with open('filter.json', 'w') as f:
f.write(
json.dumps(request.json,
ensure_ascii=False).encode('utf8'))
code = 200
return Response('', code)
def tmt_wechat_import():
items = get_collection('items')
fields = request.get_json(force=True)
if not fields:
return jsonify({'is_success': False, 'error_msg': 'ARGS ERROR'})
sources_map = [
u'腾讯科技',
u'科技每日推送',
u'科技最前线',
u'财新TMT',
u'网易科技',
u'新浪科技',
u'凤凰科技',
u'慧聪TMT',
u'蓝媒TMT',
u'蓝鲸TMT',
u'阿玻罗金融科技',
u'TMT每日观察',
u'朱劲松-TMT观察',
u'TMT观察',
u'杨吉TMT',
u'搜狐科技',
u'雷锋网',
u'36氪',
u'虎嗅网',
u'21世纪经济报道',
u'创业邦杂志',
u'中国经营报',
u'经济观察报',
u'铅笔道',
u'财新网',
u'并购汪',
u'亿欧网',
u'新智元',
u'猎云网',
u'机器之心',
u'海外情报社',
u'FT中文网',
u'界面',
u'雷帝触网',
u'好奇心日报',
u'商业周刊中文版',
u'环球老虎财经',
u'钛媒体',
u'PingWest品玩',
u'速途网',
u'第一财经',
u'秦朔朋友圈',
u'IT桔子',
u'DoNews',
u'动点科技',
u'全球企业动态',
u'蓝鲸财经网',
u'财经天下周刊',
u'VRAR创投圈',
u'蓝鲸财经记者工作平台',
u'B楼12座',
u'经纬创投',
u'小道消息',
u'VR时代',
u'财经女记者部落',
u'真格基金',
u'峰瑞资本',
u'keso怎么看',
u'一见',
]
filtered_data = {}
if fields.get('wechatName') in sources_map:
inserted = items.insert_one({
"content":
fields.get('content', ''),
"source":
u'{}__{}'.format(fields.get('wechatId', ''),
fields.get('originId', '')),
"host":
'http://mp.weixin.qq.com/',
"link":
fields.get('sourceUrl', ''),
"time":
fields.get('createdAt', ''),
"name":
u"微信 - %s" % fields.get('wechatName'),
"title":
fields.get('title', ''),
"time_human":
"",
"ctime":
int(time.time()),
"coops": ["tmt"],
})
redis = get_redis()
rt = redis.publish('news_updated', str(inserted.inserted_id))
filtered_data = {
'db_inserted_id': str(inserted.inserted_id),
'redis_published_return': rt
}
return jsonify({'is_success': True, 'filtered_data': filtered_data})
def app_push_import():
import_items = get_collection('app_push_items')
items = get_collection('items')
fields = request.get_json(force=True)
auth_key = fields.get('auth_key')
if not auth_key == '16b4af8':
return jsonify({'is_success': False, 'error_msg': 'AUTH ERROR'})
del fields['auth_key']
if not fields:
return jsonify({'is_success': False, 'error_msg': 'ARGS ERROR'})
unique_title = fields.get('title', '') + fields.get(
'packageName', 'N') + fields.get('contentText', '')
uid = hashlib.md5(unique_title.encode('utf-8')).hexdigest()
fields.update({'uid': uid})
result = import_items.update_one({'uid': uid}, {'$set': fields},
upsert=True)
if result.matched_count > 0:
return jsonify({'is_success': False, 'error_msg': 'DUPLICATED ERROR'})
sources_map = [
{
'title': u'新华社',
'packageName': "net.xinhuamm.mainclient",
'titleField': 'contentText',
'bodyField': '',
'website': 'http://www.xinhuanet.com/',
},
{
'title': u'人民日报',
'packageName': "com.peopledailychina.activity",
'titleField': 'title',
'bodyField': 'contentText',
'website': 'http://www.people.com.cn/',
},
{
'title': u'二十一财经',
'packageName': "com.twentyfirstcbh.epaper",
'titleField': 'contentText',
'bodyField': '',
'website': 'http://www.21jingji.com/',
},
]
filtered_data = {}
for source in sources_map:
if fields.get('packageName') == source.get('packageName'):
mname = 'APP_%s' % fields.get('packageName')
inserted = items.insert_one({
"content":
fields.get(source.get('bodyField'))
if source.get('bodyField') else '',
"source":
mname,
"host":
fields.get('packageName'),
"link":
source.get('website'),
"time":
fields.get('notifyTime'),
"name":
u"%s - APP推送" % source.get('title'),
"title":
fields.get(source.get('titleField')),
"time_human":
"",
"ctime":
int(time.time())
})
redis = get_redis()
rt = redis.publish('news_updated', str(inserted.inserted_id))
filtered_data = {
'db_inserted_id': str(inserted.inserted_id),
'redis_published_return': rt
}
break
return jsonify({'is_success': True, 'filtered_data': filtered_data})
def get_modules():
items = get_collection('items')
namelist = items.distinct('name')
return namelist
def sourcedefaultget():
sources = get_collection('sourceDefaultConfigs')
map = sources.find_one()
clean_id(map)
return jsonify(map)
def sourcedelete(sourceId):
sources = get_collection('sources')
is_success = sources.delete_one({'_id': ObjectId(sourceId)})
return jsonify({'is_success': is_success.acknowledged})