def send_ohlcv(self):
ohlcv_files = [ohlcv_file for ohlcv_file in os.listdir(self.OHLCV_PATH) if '.csv' in ohlcv_file]
done_count = 0
for ohlcv_file in ohlcv_files:
ticker = ohlcv_file.split('.')[0]
if OHLCV.objects.filter(code=ticker).exists():
done_count += 1
print('{} Data exists, skipping {}'.format(str(done_count), ticker))
continue
else:
df = self._retrieve_ohlcv(ohlcv_file)
ohlcv_list = []
for row_n in range(len(df)):
code = ohlcv_file.split('.')[0]
date, open_price, high_price, low_price, close_price, volume = list(df.ix[row_n])
ohlcv_inst = OHLCV(code=code,
date=str(date)[:8],
open_price=open_price,
high_price=high_price,
low_price=low_price,
close_price=close_price,
volume=volume)
ohlcv_list.append(ohlcv_inst)
OHLCV.objects.bulk_create(ohlcv_list)
## test df count and db count ##
df_len = len(df)
db_count = OHLCV.objects.filter(code=code).count()
if df_len == db_count:
done_count += 1
print('{} {} OHLCV instances successfully saved to database'.format(str(done_count), code))
else:
print('{} OHLCV instance count mismatch with the file'.format(code))
def get_ohlcv():
OPEN = False
#오늘 날짜
today_date = datetime.now().strftime('%Y%m%d')
#ticker가져오기
ticker = Ticker.objects.filter(date=today_date)
#오늘 ohlcv 가져오기(중복방지)
ohlcv = OHLCV.objects.filter(date=today_date)
#장 열었는지 확인(평일인데 안여는 날 제외하기 위해)
url = "http://finance.naver.com/item/sise_day.nhn?code=" + ticker[100].code
user_agent = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36'
}
r = requests.get(url, headers=user_agent, auth=('user', 'pass'))
d = r.text
soup = BeautifulSoup(d, "html.parser")
market_date = soup.find('span', {
'class': 'tah p10 gray03'
}).string.replace(".", "")
if (market_date == today_date):
OPEN = True
if (OPEN):
ohlcv_list = []
for i in range(len(ticker)):
#이미 저장 되었으면 pass
if not (ohlcv.filter(code=ticker[i].code).exists()):
url = "http://finance.naver.com/item/sise_day.nhn?code=" + ticker[
i].code
print(url)
df = pd.read_html(url, thousands='')
open_price = int(df[0].ix[1][3].replace(",", ""))
high_price = int(df[0].ix[1][4].replace(",", ""))
low_price = int(df[0].ix[1][5].replace(",", ""))
close_price = int(df[0].ix[1][1].replace(",", ""))
volume = int(df[0].ix[1][6].replace(",", ""))
data = OHLCV(code=ticker[i].code,
date=today_date,
open_price=open_price,
high_price=high_price,
low_price=low_price,
close_price=close_price,
volume=volume)
ohlcv_list.append(data)
print(str(i) + ' added ' + ticker[i].code + ' data')
OHLCV.objects.bulk_create(ohlcv_list)
def update_ohlcv(self):
upd_num = 0
random_ticker = Ticker.objects.all()[100].code
recent_update_date = OHLCV.objects.filter(code=random_ticker).order_by('-date').first().date
today_date = datetime.datetime.now().strftime('%Y%m%d')
if recent_update_date != today_date:
tickers = Ticker.objects.filter(date=today_date)
for ticker in tickers:
code = ticker.code
if OHLCV.objects.filter(code=code).filter(date=today_date).exists():
print('{} {} already updated. Skipping...'.format(str(upd_num), code))
upd_num += 1
continue
else:
url = "http://finance.naver.com/item/sise_day.nhn?code=" + code
print('{} {}'.format(str(upd_num), url))
df = pd.read_html(url, thousands='')
df = df[0]
ohlcv_list = []
index = 1
while index:
try:
date = str(df.ix[index][0].replace(".", ""))
if date == recent_update_date:
break
else:
open_price = int(df.ix[index][3].replace(",", ""))
high_price = int(df.ix[index][4].replace(",", ""))
low_price = int(df.ix[index][5].replace(",", ""))
close_price = int(df.ix[index][1].replace(",", ""))
volume = int(df.ix[index][6].replace(",", ""))
data = OHLCV(code=code,
date=date,
open_price=open_price,
high_price=high_price,
low_price=low_price,
close_price=close_price,
volume=volume)
ohlcv_list.append(data)
print(str(upd_num)+ ' added ' + code + ' data')
index += 1
except:
break
OHLCV.objects.bulk_create(ohlcv_list)
upd_num += 1
def send_bm(self):
df = self._retrieve_bm()
code = 'BM'
ohlcv_list = []
for row_n in range(len(df)):
date, open_price, high_price, low_price, close_price, adj_close_price, volume = list(df.ix[row_n])
ohlcv_inst = OHLCV(code=code,
date=str(date).replace('-', '')[:8],
open_price=open_price,
high_price=high_price,
low_price=low_price,
close_price=adj_close_price,
volume=volume)
ohlcv_list.append(ohlcv_inst)
OHLCV.objects.bulk_create(ohlcv_list)
## test df count and db count ##
df_len = len(df)
db_count = OHLCV.objects.filter(code=code).count()
if df_len == db_count:
print('{} OHLCV instances successfully saved to database'.format(code))
else:
print('{} OHLCV instance count mismatch with the file'.format(code))
import pandas_datareader as wb
import datetime
from restapi.models import Ticker, OHLCV
start = datetime.datetime(1990, 1, 1)
end = datetime.datetime(2017, 12, 11)
df_null = wb.DataReader("^KS11", "yahoo", start, end)
df = df_null.dropna()
code = Ticker.objects.filter(code='BM').first()
for i in range(len(df)):
row = df.ix[i]
date = row.name.strftime('%Y%m%d')
open_p = int(row['Open'])
high = int(row['High'])
low = int(row['Low'])
close = int(row['Adj Close'])
volume = int(row['Volume'])
data = OHLCV(code=code,
date=date,
open_price=open_p,
high_price=high,
low_price=low,
close_price=close,
volume=volume)
data.save()
print(date + ' saved')
break
date = df[0].ix[i][0].replace(".", "")
if(date[0:4] == '1999'):
date_1999 = True
break
elif (int(date[0:8])>int(today_date)): #지정 날짜 부터 저장하기
continue
else:
open_price = int(df[0].ix[i][3].replace(",", ""))
high_price = int(df[0].ix[i][4].replace(",", ""))
low_price = int(df[0].ix[i][5].replace(",", ""))
close_price = int(df[0].ix[i][1].replace(",", ""))
volume = int(df[0].ix[i][6].replace(",", ""))
#code = Ticker.objects.filter(id=ticker['id'][t]).first()
#db에 저장
data = OHLCV(code=ticker[t], date=date, open_price=open_price, high_price=high_price, low_price=low_price, close_price=close_price, volume=volume)
data.save()
else:
for i in range(1,11):
if(str(df[0].ix[i][0]) == "nan"):
continue
date = df[0].ix[i][0].replace(".", "")
if(date[0:4] == '1999'):
date_1999 = True
break
elif (int(date[0:8])>int(today_date)): #지정 날짜 부터 저장하기
continue
else:
open_price = int(df[0].ix[i][3].replace(",", ""))
high_price = int(df[0].ix[i][4].replace(",", ""))