def close_price(secu, trade_date, typ):
# mysql = getattr(query_string, typ)
from eggs.db.mysql_client import MySQLClient
mysql = MySQLClient("192.168.251.95", "python_team", "python_team",
"ada-fd")
sql = "SELECT close_price from equity_price where secu_code=%s and trade_date <=%s" \
"order by trade_date DESC limit 1"
if secu:
price = mysql.get(sql, *(secu, trade_date))
if price:
return price[0]
def close_price(secu, trade_date, typ):
# mysql = getattr(query_string, typ)
from eggs.db.mysql_client import MySQLClient
mysql = MySQLClient("192.168.251.95", "python_team", "python_team", "ada-fd")
sql = (
"SELECT close_price from equity_price where secu_code=%s and trade_date <=%s" "order by trade_date DESC limit 1"
)
if secu:
price = mysql.get(sql, *(secu, trade_date))
if price:
return price[0]
def insert_db(self, total_data):
coll_in = Mongodb('192.168.251.95', 27017, 'ada', 'base_margin_trading')
coll_stock = Mongodb('192.168.251.95', 27017, 'ada', 'base_stock')
coll_fund = Mongodb('192.168.251.95', 27017, 'fund', 'base_fund')
sql_db = MySQLClient("192.168.251.95", "python_team", "python_team", "ada-fd")
print '\tnow start to insert mongodb, waiting......'
d = (lambda v: '%.4f' % float(v))
for pdt in total_data:
# 信用交易日期 标的证券代码 标的证券简称 本日融资余额(元) 本日融资买入额(元)
# 本日融资偿还额(元) 本日融券余量 本日融券卖出量 本日融券偿还量
secu_cd = secu_code(pdt[1], coll_stock, coll_fund)
trade_date = '-'.join([pdt[0][:4], pdt[0][4:6], pdt[0][6:]])
uid = str(uuid.uuid5(uuid.NAMESPACE_DNS, ''.join(self._valid(pdt)).encode('u8')))
data = {
'secu': secu_cd or pdt[1], 'date': trade_date, 'total': d(int(pdt[3])), 'stat': 2,
'typ': 'sha', 'crt': datetime.now(), 'uuid': uid,
'fi': {
'ba': d(pdt[3]),
'bu': d(pdt[4]),
're': d(pdt[5])
},
'se': {
'ba': '0.0000',
'ma': d(pdt[6]),
'so': d(pdt[7]),
're': d(pdt[8])
},
'upt': datetime.now()
}
if coll_in.get({'uuid': uid, 'typ': 'sha'}, {'secu': 1}):
continue
elif secu_cd is None:
coll_in.insert(data)
else:
seba = sha_seba(secu_cd, pdt[6], trade_date, sql_db)
if seba is not None:
data['total'] = d(int(pdt[3]) + seba)
data['se']['ba'] = d(seba)
coll_in.insert(data)
coll_in.disconnect()
coll_stock.disconnect()
sql_db.disconnect()
print '\tinsert all done!'
def get_data():
sorted_type = [('appchina', 5), ('app_mi', 6), ('jifeng', 8)]
command_type = [('casperjs login_app.js {0}'.format(app_typ[0]), app_typ[1]) for app_typ in sorted_type]
pool = TheadPool(8)
rets = pool.map(static_app_count, [CONFIG[k] for k in CONFIG])
rets.extend(pool.map(login_app_count, command_type))
pool.close()
db = MySQLClient('122.144.134.3', 'ada_user', 'ada_user', 'invest')
sql = "INSERT INTO mobile_app_count(typ, platform, total_count, day_count, upt) values(%s, %s, %s, %s, NOW())"
platform = {1: u'百度手机助手', 2: u'安卓市场', 3: u'91手机助手', 4: u'应用宝', 5: u'掌上应用汇',
6: u'小米', 7: u'360手机助手', 8: u'机锋市场', 9: u'豌豆荚', 10: u'华为应用'}
for ty, num in rets:
day_count, total_count = num if isinstance(num, tuple) else ('', num or '')
print ty, platform.get(ty), day_count, total_count
db.execute(sql, *(ty, platform.get(ty), total_count, day_count))
db.disconnect()
def get_data():
sorted_type = [('appchina', 5), ('app_mi', 6), ('jifeng', 8)]
command_type = [('casperjs login_app.js {0}'.format(app_typ[0]),
app_typ[1]) for app_typ in sorted_type]
pool = TheadPool(8)
rets = pool.map(static_app_count, [CONFIG[k] for k in CONFIG])
rets.extend(pool.map(login_app_count, command_type))
pool.close()
db = MySQLClient('122.144.134.3', 'ada_user', 'ada_user', 'invest')
sql = "INSERT INTO mobile_app_count(typ, platform, total_count, day_count, upt) values(%s, %s, %s, %s, NOW())"
platform = {
1: u'百度手机助手',
2: u'安卓市场',
3: u'91手机助手',
4: u'应用宝',
5: u'掌上应用汇',
6: u'小米',
7: u'360手机助手',
8: u'机锋市场',
9: u'豌豆荚',
10: u'华为应用'
}
for ty, num in rets:
day_count, total_count = num if isinstance(num, tuple) else ('',
num or '')
print ty, platform.get(ty), day_count, total_count
db.execute(sql, *(ty, platform.get(ty), total_count, day_count))
db.disconnect()
base_url = 'http://www.szse.cn/szseWeb/FrontController.szse?randnum=&'
query_string = 'ACTIONID=7&AJAX=AJAX-TRUE&CATALOGID=1265_xyjy&TABKEY=tab1&tab1PAGECOUNT={0}&' \
'tab1RECORDCOUNT={1}&REPORT_ACTION=navigate&tab1PAGENUM={2}'
bond_string = 'ACTIONID=7&AJAX=AJAX-TRUE&CATALOGID=dzjy_xyjy&TABKEY=tab1&tab1PAGECOUNT={0}&' \
'tab1RECORDCOUNT={1}&REPORT_ACTION=navigate&tab1PAGENUM={2}'
user_agent = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'
referer = 'http://www.sse.com.cn/disclosure/diclosure/block/deal/'
stock_fund_url = 'http://query.sse.com.cn/commonQuery.do?&jsonCallBack=&isPagination=true&' \
'sqlId=COMMON_SSE_XXPL_JYXXPL_DZJYXX_L_1&stockId=&startDate=%s&endDate=%s&' \
'pageHelp.pageSize=15&pageHelp.pageNo=1&pageHelp.beginPage=%s&pageHelp.endPage=5&' \
'pageHelp.cacheSize=1'
bond_url = 'http://query.sse.com.cn/commonQuery.do?&jsonCallBack=&isPagination=true&' \
'sqlId=COMMON_SSE_XXPL_JYXXPL_DZJYXX_L_2&stockId=&startDate=%s&endDate=%s&' \
'pageHelp.pageSize=15&pageHelp.pageNo=1&pageHelp.beginPage=%s&pageHelp.endPage=5&' \
'pageHelp.cacheSize=1'
coll_in = Mongodb('192.168.251.95', 27017, 'ada', 'base_block_trade')
coll_stock = Mongodb('192.168.251.95', 27017, 'ada', 'base_stock')
coll_fund = Mongodb('192.168.251.95', 27017, 'fund', 'base_fund')
coll_bond = Mongodb('192.168.251.95', 27017, 'ada', 'base_bond')
coll_vary = Mongodb('192.168.251.95', 27017, 'ada', 'base_share_vary')
mysql = MySQLClient("192.168.251.95", "python_team", "python_team", "ada-fd")
sha_command_history = r'casperjs D:\project\autumn\crawler\block_trade\block_trade_with_date.js ' \
r'--st_date={0} --ed_date={1} --outfile={0}'
sha_command_update = r'casperjs block_trade.js'