def crawlEastMoney():
db.initDb()
etfListUrl="http://fund.eastmoney.com/ETFN_jzzzl.html"
eftHistUrl="http://api.fund.eastmoney.com/f10/lsjz?callback={0}&fundCode={1}&pageIndex=1&pageSize={3}&startDate=&endDate=&_={2}"
callback='jQuery18303575276080195784_1602819402553'
crawler=Crawler(**{'referer':'http://fundf10.eastmoney.com/'})
# 获取etf列表
html=crawler.downloadHtml(etfListUrl)
items=cleaner.getEtfList(html)
db.insertList(items)
# 插入etf历史净值
db.clear('fund_hist')
for item in items:
code=item['code']
print(code)
ts=int(time.time())
# 获取历史净值记录条数
url=eftHistUrl.format(callback,code,ts,1)
content=crawler.downloadHtml(url)
size=cleaner.getHistCount(content,callback)
# 一次性获取所有历史净值
url=eftHistUrl.format(callback,code,ts,size)
content=crawler.downloadHtml(url)
items=cleaner.getHistList(content,callback,code)
db.insertHistList(items)
def setup_method(self, method):
db.clear()
db.drivers.autoregister_class(TestDriver)
db.from_url(MEM_URL)
db.do(CREATE_FOO_SQL)
db.do("INSERT INTO foo VALUES (1, 'foo')")
self.db = db.get()
def setup_method(self, method):
db.clear()
db.drivers.autoregister_class(TestDriver)
db.from_url(MEM_URL)
db.do(CREATE_FOO_SQL)
db.do("INSERT INTO foo VALUES (1, 'foo')")
self.db = db.get()
def setup_method(self, method):
db.clear()
import db_psycopg2
db.from_environ("DB_PSYCOPG2_TEST_URL")
self.db = db.get()
db.do("""CREATE OR REPLACE FUNCTION always1() RETURNS INTEGER AS $$
BEGIN
RETURN 1;
END
$$ LANGUAGE 'plpgsql';
""")
def setup_method(self, method):
db.clear()
import db_psycopg2
db.from_environ("DB_PSYCOPG2_TEST_URL")
self.db = db.get()
db.do("""CREATE OR REPLACE FUNCTION always1() RETURNS INTEGER AS $$
BEGIN
RETURN 1;
END
$$ LANGUAGE 'plpgsql';
""")
def test_dump():
o = BaseObject('Base')
m = MobObject('Mob')
p = PlayerObject('Player')
r = RoomObject('Room')
db.dump(fname)
l = len(db.objects)
logger.info('Current objects: %s.', db.objects)
db.clear()
assert not len(db.objects)
db.load(fname)
assert len(db.objects) >= l, '%s (%s -> %s).' % (db.objects, len(db.objects), l)
os.remove(fname)
logger.info('Removed DB file: %s.', fname)
def install_one_default_driver(self):
db.clear()
self.default_db = db.register(self.make_driver("connect_default_only"))
def install_one_driver(self):
db.clear()
self.only_db = db.register(self.make_driver("connect_one_only"),
db_name="only")
def commander(command):
command_ = command.split(' ')
if command_[0] == 'insert':
db_name = command_[2]
value_1 = command_[-1]
value_2 = command_[-2]
value_3 = command_[-3]
data = {}
if check_value(value_1):
value_ = value_1.split('=')
data[value_[0]] = value_[1]
if check_value(value_2):
value_ = value_2.split('=')
data[value_[0]] = value_[1]
if check_value(value_3):
value_ = value_3.split('=')
data[value_[0]] = value_[1]
function_name = 'create_' + db_name
result = eval(function_name + "(data)")
db_file_name = db_name + '.bin'
db_data = read(db_file_name)
db_data.append(result)
write(db_data, db_file_name)
return 'done'
if command_[0] == 'update':
db_name = command_[1]
query_value = command_[-1]
update_value = command_[-3]
query_data = []
update_data = []
if check_value(query_value):
query_data = query_value.split('=')
if check_value(update_value):
update_data = update_value.split('=')
db_file_name = db_name + '.bin'
db_data = read(db_file_name)
for item in db_data:
if 'code' in query_data[0]:
query_data[1] = int(query_data[1])
if 'code' in update_data[0]:
update_data[1] = int(update_data[1])
if item[query_data[0]] == query_data[1]:
item[update_data[0]] = update_data[1]
clear(db_file_name)
write(db_data, db_file_name)
return 'done'
if command_[0] == 'delete':
db_name = command_[2]
value_1 = command_[-1]
value_2 = command_[-3]
value1 = []
value2 = []
if check_value(value_1):
value1 = value_1.split('=')
if check_value(value_2):
value2 = value_2.split('=')
db_file_name = db_name + '.bin'
db_data = read(db_file_name)
for item in db_data:
if 'code' in value1[0]:
value1[1] = int(value1[1])
if 'code' in value2[0]:
value2[1] = int(value2[1])
if item[value1[0]] == value1[1] and item[value2[0]] == value2[1]:
item['active'] = 0
clear(db_file_name)
write(db_data, db_file_name)
return 'done'
if command_[0] == 'select':
db_name = command_[2]
value_1 = command_[-1]
value_ = []
if check_value(value_1):
value_ = value_1.split('=')
db_file_name = db_name + '.bin'
db_data = read(db_file_name)
query = ''
for item in db_data:
if 'code' in value_[0]:
value_[1] = int(value_[1])
switch = ''
if value_[0] == 'st_code':
switch = 'course_code'
else:
switch = 'st_code'
if item[value_[0]] == value_[1]:
function_name = 'get_' + db_name
result = eval(function_name + "(item, switch)")
query += result + '\n'
return query
def clear(self):
r = requests.get(telegram_bot_url + telegram_token + "/setWebhook",
params={'url': ""})
print r
clear()
def install_one_default_driver(self):
db.clear()
self.default_db = db.register(self.make_driver("connect_default_only"))
def test_not_already_clear(self):
self.install_one_driver()
db.clear()
assert db.count_dbs() == 0
def install_two_drivers(self):
db.clear()
self.first_db = db.register(self.make_driver("connect_two_first"),
db_name="first")
self.second_db = db.register(self.make_driver("connect_two_second"),
db_name="second")
def test_not_already_clear(self):
self.install_one_driver()
db.clear()
assert db.count_dbs() == 0
for song in songs:
name = song['track']['artistName']
info = spotify.get_artistByName(name)
if (info != {}):
artists.append({
'id': info['artists']['items'][0]['id'],
'name': name,
'genres': info['artists']['items'][0]['genres']
})
return artists
if (get_newData):
print('Cleaning collections...')
db.clear()
artists = []
print('Getting artists list...')
for playlist in helpers.load_data('playlist')[0]['playlists']:
get_artists(artists, playlist['items'])
db.add_artists(artists)
print('Done getting artists!')
streaming_history = []
top_artists = []
top_genres = []
print('Getting streaming history...')
def setUp(self):
app.testing = True
self.app = app.test_client()
db.read()
db.clear()
def delete():
user = request.args['user']
print(user)
db.clear(user)
return redirect(url_for("logout"))
def test_not_already_clear_multiple(self):
self.install_two_drivers()
db.clear()
assert db.count_dbs() == 0
def install_two_drivers(self):
db.clear()
self.first_db = db.register(self.make_driver("connect_two_first"),
db_name="first")
self.second_db = db.register(self.make_driver("connect_two_second"),
db_name="second")
def install_one_driver(self):
db.clear()
self.only_db = db.register(self.make_driver("connect_one_only"),
db_name="only")
def test_already_clear(self):
self.clear_drivers()
db.clear()
assert db.count_dbs() == 0
def test_already_clear(self):
self.clear_drivers()
db.clear()
assert db.count_dbs() == 0
def test_not_already_clear_multiple(self):
self.install_two_drivers()
db.clear()
assert db.count_dbs() == 0
def clear(self):
r = requests.get( telegram_bot_url + telegram_token + "/setWebhook", params={'url': ""} )
print r
clear()
def clear(self):
for db in self.dbs:
db.clear()
def clear(self):
for db in self.dbs:
db.clear()
