def get_data(): sorted_type = [('appchina', 5), ('app_mi', 6), ('jifeng', 8)] command_type = [('casperjs login_app.js {0}'.format(app_typ[0]), app_typ[1]) for app_typ in sorted_type] pool = TheadPool(8) rets = pool.map(static_app_count, [CONFIG[k] for k in CONFIG]) rets.extend(pool.map(login_app_count, command_type)) pool.close() db = MySQLClient('122.144.134.3', 'ada_user', 'ada_user', 'invest') sql = "INSERT INTO mobile_app_count(typ, platform, total_count, day_count, upt) values(%s, %s, %s, %s, NOW())" platform = { 1: u'百度手机助手', 2: u'安卓市场', 3: u'91手机助手', 4: u'应用宝', 5: u'掌上应用汇', 6: u'小米', 7: u'360手机助手', 8: u'机锋市场', 9: u'豌豆荚', 10: u'华为应用' } for ty, num in rets: day_count, total_count = num if isinstance(num, tuple) else ('', num or '') print ty, platform.get(ty), day_count, total_count db.execute(sql, *(ty, platform.get(ty), total_count, day_count)) db.disconnect()
def insert_db(self, total_data): coll_in = Mongodb('192.168.251.95', 27017, 'ada', 'base_margin_trading') coll_stock = Mongodb('192.168.251.95', 27017, 'ada', 'base_stock') coll_fund = Mongodb('192.168.251.95', 27017, 'fund', 'base_fund') sql_db = MySQLClient("192.168.251.95", "python_team", "python_team", "ada-fd") print '\tnow start to insert mongodb, waiting......' d = (lambda v: '%.4f' % float(v)) for pdt in total_data: # 信用交易日期 标的证券代码 标的证券简称 本日融资余额(元) 本日融资买入额(元) # 本日融资偿还额(元) 本日融券余量 本日融券卖出量 本日融券偿还量 secu_cd = secu_code(pdt[1], coll_stock, coll_fund) trade_date = '-'.join([pdt[0][:4], pdt[0][4:6], pdt[0][6:]]) uid = str(uuid.uuid5(uuid.NAMESPACE_DNS, ''.join(self._valid(pdt)).encode('u8'))) data = { 'secu': secu_cd or pdt[1], 'date': trade_date, 'total': d(int(pdt[3])), 'stat': 2, 'typ': 'sha', 'crt': datetime.now(), 'uuid': uid, 'fi': { 'ba': d(pdt[3]), 'bu': d(pdt[4]), 're': d(pdt[5]) }, 'se': { 'ba': '0.0000', 'ma': d(pdt[6]), 'so': d(pdt[7]), 're': d(pdt[8]) }, 'upt': datetime.now() } if coll_in.get({'uuid': uid, 'typ': 'sha'}, {'secu': 1}): continue elif secu_cd is None: coll_in.insert(data) else: seba = sha_seba(secu_cd, pdt[6], trade_date, sql_db) if seba is not None: data['total'] = d(int(pdt[3]) + seba) data['se']['ba'] = d(seba) coll_in.insert(data) coll_in.disconnect() coll_stock.disconnect() sql_db.disconnect() print '\tinsert all done!'
def get_data(): sorted_type = [('appchina', 5), ('app_mi', 6), ('jifeng', 8)] command_type = [('casperjs login_app.js {0}'.format(app_typ[0]), app_typ[1]) for app_typ in sorted_type] pool = TheadPool(8) rets = pool.map(static_app_count, [CONFIG[k] for k in CONFIG]) rets.extend(pool.map(login_app_count, command_type)) pool.close() db = MySQLClient('122.144.134.3', 'ada_user', 'ada_user', 'invest') sql = "INSERT INTO mobile_app_count(typ, platform, total_count, day_count, upt) values(%s, %s, %s, %s, NOW())" platform = {1: u'百度手机助手', 2: u'安卓市场', 3: u'91手机助手', 4: u'应用宝', 5: u'掌上应用汇', 6: u'小米', 7: u'360手机助手', 8: u'机锋市场', 9: u'豌豆荚', 10: u'华为应用'} for ty, num in rets: day_count, total_count = num if isinstance(num, tuple) else ('', num or '') print ty, platform.get(ty), day_count, total_count db.execute(sql, *(ty, platform.get(ty), total_count, day_count)) db.disconnect()