This repository has been archived by the owner on Jun 29, 2020. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
manager_flask_app.py
143 lines (125 loc) · 5.8 KB
/
manager_flask_app.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
# coding:utf-8
from __future__ import absolute_import, unicode_literals
from flask_script import Manager, Shell, Server
from web.extension import db, mongo, redis
from web.app import create_app
import datetime
import decimal
import requests
__author__ = "golden"
__date__ = '2017/2/7'
app = create_app('web_config.Developer')
manager = Manager(app)
all_area = ['武侯区', '青羊区', '高新区', '天府新区', '金牛区', '锦江区', '成华区', '双流区', '温江区', '龙泉区', '郫县区', '新都区', '都江堰区', '高新西区', '成都周边']
def make_shell_context():
return dict(mongo=mongo, app=app)
@manager.option('-d', '--date', dest='date', help=u'统计日期之后', required=False)
@manager.option('-s', '--site', dest='site', help=u'网站', required=False)
def day_increase(date=None, site=None):
"""统计每日新增数据"""
if not date:
date = str((datetime.datetime.now() - datetime.timedelta(days=3)).date())
if site:
sites = [site]
else:
sites = ['fang', 'lianjia', 'anjuke']
for s in sites:
mongo_db = getattr(mongo.db, s)
datas = mongo_db.group(key={'publish_date': True}, initial={'count': 0},
reduce="function(doc,prev){prev.count++}",
condition={'publish_date': {'$gte': date}})
for data in datas:
mongo.db.day_increase.update({"publish_date": data['publish_date'],
"domain": s}, {"$set": data},
upsert=True)
print("%s done!" % s)
@manager.option('-d', '--date', dest='date', help=u'统计日期之后', required=False)
@manager.option('-s', '--site', dest='site', help=u'网站', required=False)
def average_price_area(date=None, site=None):
"""按区域统计每平米平均价格"""
if not date:
date = str((datetime.datetime.now() - datetime.timedelta(days=3)).date())
if site:
sites = [site]
else:
sites = ['fang', 'lianjia', 'anjuke']
for s in sites:
mongo_db = getattr(mongo.db, s)
datas = mongo_db.group(key={'publish_date': True, 'area': True}, initial={'count': 0, 'sum': 0},
reduce="function(doc,prev){prev.count++,prev.sum+=parseFloat(doc.m2_price)}",
condition={'m2_price': {'$exists': True}, 'publish_date': {'$gte': date}})
for data in datas:
if data['zone'] not in all_area and not data['zone'].endswith('区'):
data['zone'] += '区'
if data['zone'] in all_area:
_avg = data['sum'] / data['count']
avg = decimal.Decimal(_avg).quantize(decimal.Decimal('0.00'))
data.pop('count')
data.pop('sum')
data['average'] = unicode(avg)
mongo.db.average_price_area.update({"publish_date": data['publish_date'],
'zone': data['zone'],
'domain': s}, {"$set": data},
upsert=True)
print("%s done!" % s)
@manager.option('-d', '--date', dest='date', help=u'统计日期之后', required=False)
@manager.option('-s', '--site', dest='site', help=u'网站', required=False)
def average_price_day(date=None, site=None):
"""每天平均数"""
if not date:
date = str((datetime.datetime.now() - datetime.timedelta(days=3)).date())
if site:
sites = [site]
else:
sites = ['fang', 'lianjia', 'anjuke']
for s in sites:
mongo_db = getattr(mongo.db, s)
datas = mongo_db.group(key={'publish_date': True, }, initial={'count': 0, 'sum': 0},
reduce="function(doc,prev){prev.count++,prev.sum+=parseFloat(doc.m2_price)}",
condition={'m2_price': {'$exists': True}, 'publish_date': {'$gte': date}})
for data in datas:
_avg = data['sum'] / data['count']
avg = decimal.Decimal(_avg).quantize(decimal.Decimal('0.00'))
data.pop('count')
data.pop('sum')
data['average'] = unicode(avg)
mongo.db.average_price_day.update({"publish_date": data['publish_date'],
"domain": s}, {"$set": data},
upsert=True)
print("%s done!" % s)
@manager.command
def delete_history(before_date=None):
"""删除esf历史数据"""
if not before_date:
before_date = str((datetime.datetime.now() - datetime.timedelta(days=10)).date())
for s in ['fang', 'lianjia', 'anjuke']:
mongo_db = getattr(mongo.db, s)
mongo_db.remove({"publish_date": {"$lt": before_date}})
return 'success'
@manager.command
def check_proxy():
all_proxy = mongo.db.proxy.find()
for proxy in all_proxy:
proxies = {"http": "http://%s:%s" % (proxy['ip'], proxy['port'])}
try:
res = requests.get('http://www.ip138.com/', proxies=proxies, timeout=2)
if res.ok:
proxy['check_time'] = str(datetime.datetime.now())
proxy['checked'] = 1
mongo.db.proxy.update({"_id": proxy['_id']}, {"$set": proxy},
upsert=True)
else:
mongo.db.proxy.remove({"_id": proxy['_id']})
print('delete : %s' % proxy['ip'])
except:
mongo.db.proxy.remove({"_id": proxy['_id']})
print('delete : %s' % proxy['ip'])
manager.add_command("shell", Shell(make_context=make_shell_context, use_bpython=False))
manager.add_command("runserver", Server(
use_debugger=True,
use_reloader=True,
port=5001,
host="0.0.0.0"
))
if __name__ == "__main__":
manager.run()