def crawlEastMoney(): db.initDb() etfListUrl="http://fund.eastmoney.com/ETFN_jzzzl.html" eftHistUrl="http://api.fund.eastmoney.com/f10/lsjz?callback={0}&fundCode={1}&pageIndex=1&pageSize={3}&startDate=&endDate=&_={2}" callback='jQuery18303575276080195784_1602819402553' crawler=Crawler(**{'referer':'http://fundf10.eastmoney.com/'}) # 获取etf列表 html=crawler.downloadHtml(etfListUrl) items=cleaner.getEtfList(html) db.insertList(items) # 插入etf历史净值 db.clear('fund_hist') for item in items: code=item['code'] print(code) ts=int(time.time()) # 获取历史净值记录条数 url=eftHistUrl.format(callback,code,ts,1) content=crawler.downloadHtml(url) size=cleaner.getHistCount(content,callback) # 一次性获取所有历史净值 url=eftHistUrl.format(callback,code,ts,size) content=crawler.downloadHtml(url) items=cleaner.getHistList(content,callback,code) db.insertHistList(items)
def setup_method(self, method): db.clear() db.drivers.autoregister_class(TestDriver) db.from_url(MEM_URL) db.do(CREATE_FOO_SQL) db.do("INSERT INTO foo VALUES (1, 'foo')") self.db = db.get()
def setup_method(self, method): db.clear() import db_psycopg2 db.from_environ("DB_PSYCOPG2_TEST_URL") self.db = db.get() db.do("""CREATE OR REPLACE FUNCTION always1() RETURNS INTEGER AS $$ BEGIN RETURN 1; END $$ LANGUAGE 'plpgsql'; """)
def test_dump(): o = BaseObject('Base') m = MobObject('Mob') p = PlayerObject('Player') r = RoomObject('Room') db.dump(fname) l = len(db.objects) logger.info('Current objects: %s.', db.objects) db.clear() assert not len(db.objects) db.load(fname) assert len(db.objects) >= l, '%s (%s -> %s).' % (db.objects, len(db.objects), l) os.remove(fname) logger.info('Removed DB file: %s.', fname)
def install_one_default_driver(self): db.clear() self.default_db = db.register(self.make_driver("connect_default_only"))
def install_one_driver(self): db.clear() self.only_db = db.register(self.make_driver("connect_one_only"), db_name="only")
def commander(command): command_ = command.split(' ') if command_[0] == 'insert': db_name = command_[2] value_1 = command_[-1] value_2 = command_[-2] value_3 = command_[-3] data = {} if check_value(value_1): value_ = value_1.split('=') data[value_[0]] = value_[1] if check_value(value_2): value_ = value_2.split('=') data[value_[0]] = value_[1] if check_value(value_3): value_ = value_3.split('=') data[value_[0]] = value_[1] function_name = 'create_' + db_name result = eval(function_name + "(data)") db_file_name = db_name + '.bin' db_data = read(db_file_name) db_data.append(result) write(db_data, db_file_name) return 'done' if command_[0] == 'update': db_name = command_[1] query_value = command_[-1] update_value = command_[-3] query_data = [] update_data = [] if check_value(query_value): query_data = query_value.split('=') if check_value(update_value): update_data = update_value.split('=') db_file_name = db_name + '.bin' db_data = read(db_file_name) for item in db_data: if 'code' in query_data[0]: query_data[1] = int(query_data[1]) if 'code' in update_data[0]: update_data[1] = int(update_data[1]) if item[query_data[0]] == query_data[1]: item[update_data[0]] = update_data[1] clear(db_file_name) write(db_data, db_file_name) return 'done' if command_[0] == 'delete': db_name = command_[2] value_1 = command_[-1] value_2 = command_[-3] value1 = [] value2 = [] if check_value(value_1): value1 = value_1.split('=') if check_value(value_2): value2 = value_2.split('=') db_file_name = db_name + '.bin' db_data = read(db_file_name) for item in db_data: if 'code' in value1[0]: value1[1] = int(value1[1]) if 'code' in value2[0]: value2[1] = int(value2[1]) if item[value1[0]] == value1[1] and item[value2[0]] == value2[1]: item['active'] = 0 clear(db_file_name) write(db_data, db_file_name) return 'done' if command_[0] == 'select': db_name = command_[2] value_1 = command_[-1] value_ = [] if check_value(value_1): value_ = value_1.split('=') db_file_name = db_name + '.bin' db_data = read(db_file_name) query = '' for item in db_data: if 'code' in value_[0]: value_[1] = int(value_[1]) switch = '' if value_[0] == 'st_code': switch = 'course_code' else: switch = 'st_code' if item[value_[0]] == value_[1]: function_name = 'get_' + db_name result = eval(function_name + "(item, switch)") query += result + '\n' return query
def clear(self): r = requests.get(telegram_bot_url + telegram_token + "/setWebhook", params={'url': ""}) print r clear()
def test_not_already_clear(self): self.install_one_driver() db.clear() assert db.count_dbs() == 0
def install_two_drivers(self): db.clear() self.first_db = db.register(self.make_driver("connect_two_first"), db_name="first") self.second_db = db.register(self.make_driver("connect_two_second"), db_name="second")
for song in songs: name = song['track']['artistName'] info = spotify.get_artistByName(name) if (info != {}): artists.append({ 'id': info['artists']['items'][0]['id'], 'name': name, 'genres': info['artists']['items'][0]['genres'] }) return artists if (get_newData): print('Cleaning collections...') db.clear() artists = [] print('Getting artists list...') for playlist in helpers.load_data('playlist')[0]['playlists']: get_artists(artists, playlist['items']) db.add_artists(artists) print('Done getting artists!') streaming_history = [] top_artists = [] top_genres = [] print('Getting streaming history...')
def setUp(self): app.testing = True self.app = app.test_client() db.read() db.clear()
def delete(): user = request.args['user'] print(user) db.clear(user) return redirect(url_for("logout"))
def test_not_already_clear_multiple(self): self.install_two_drivers() db.clear() assert db.count_dbs() == 0
def test_already_clear(self): self.clear_drivers() db.clear() assert db.count_dbs() == 0
def clear(self): r = requests.get( telegram_bot_url + telegram_token + "/setWebhook", params={'url': ""} ) print r clear()
def clear(self): for db in self.dbs: db.clear()