def submit(self):
self.download_label.hide()
country = self.input_country.text()
self.input_country.setText('')
try:
response = get_data(country)
except requests.exceptions.RequestException:
print('Check your Internet connection!')
response = None
if response is not None:
try:
data = response.json()
newest_data = data[-1]
except KeyError:
self.create_error_label()
self.info_label.hide()
self.delete_btn.hide()
return
else:
title, date, message = output_message(newest_data)
self.info_label.show()
self.create_info_label(title, date, message)
self.create_delete_btn()
self.create_download_btn(data)
self.error_label.setText('')
def _parse_data_and_insert_db(self, table_name, data):
ret_array = fetch_data.get_data(fetch_data.regular_split("hq_sinajs_cn_list", data))
if 0 == len(ret_array):
LOG_WARNING("hg_sinajs_cn_list regular %s empty data" %(data))
return
data_array = []
into_db_columns = ["share_id","share_name","today_open","yesterday_close","cur","today_high","today_low","compete_buy_price","compete_sale_price",
"trade_num","trade_price","buy_1_num","buy_1_price","buy_2_num","buy_2_price","buy_3_num","buy_3_price","buy_4_num","buy_4_price",
"buy_5_num","buy_5_price","sale_1_num","sale_1_price","sale_2_num","sale_2_price","sale_3_num","sale_3_price","sale_4_num","sale_4_price",
"sale_5_num","sale_5_price","time_date_str","time_str","empty"]
columns_count = len(into_db_columns)
for item in ret_array:
if len(item) != columns_count:
LOG_INFO("%s length is not match for column length %d" %(str(item), columns_count))
continue
data_array.append(item)
if 0 == len(data_array):
return
db_manager = mysql_manager()
conn = db_manager.get_mysql_conn(self._conn_name)
conn.insert_data(table_name ,into_db_columns, data_array)
def _get_all_share_ids(self):
date_info = time.strftime('%Y_%m_%d')
trade_table_name = "trade_info_%s" % (date_info)
share_ids = fetch_data.get_data(fetch_data.select_db("daily_temp", trade_table_name, ["share_id"],{"share_id":[["000001","000010","000301","000601","000901","002101","002401","002701","300001","300301","600301","600601","601801","603001","603601","603901"],"in"]}, pre = "distinct"))
#share_ids = fetch_data.get_data(fetch_data.select_db("daily_temp", trade_table_name, ["share_id"],{}, pre = "distinct"))
#share_ids = fetch_data.get_data(fetch_data.select_db("daily_temp", trade_table_name, ["share_id"],{"share_id":["000001","="]}, pre = "distinct"))
return share_ids
def _get_all_share_ids(self):
date_info = time.strftime('%Y_%m_%d')
trade_table_name = "trade_info_%s" % (date_info)
share_ids = fetch_data.get_data(
fetch_data.select_db(self._daily_temp_conn_name,
trade_table_name, ["share_id"], {},
pre="distinct"))
return share_ids
def _get_all_share_ids(self):
date_info = time.strftime('%Y_%m_%d')
trade_table_name = "trade_info_%s" % (date_info)
#share_ids = fetch_data.get_data(fetch_data.select_db("daily_temp", trade_table_name, ["share_id"],{"share_id":["000001","="]}, pre = "distinct"))
share_ids = fetch_data.get_data(
fetch_data.select_db("daily_temp",
trade_table_name, ["share_id"], {},
pre="distinct"))
return share_ids
def _get_daily_info(self, share_id, table_name, start_time, compare):
stock_conn_manager_obj = stock_conn_manager()
conn_name = stock_conn_manager_obj.get_conn_name(share_id)
data = fetch_data.get_data(
fetch_data.select_db(conn_name,
table_name,
self._table_keys,
{"time": [start_time, compare]},
extend="order by time desc"))
return data
def _get_all_share_ids_market_type(self):
share_ids = self._get_all_share_ids()
ids = []
for share_id in share_ids:
ids.append(share_id[0])
share_ids = fetch_data.get_data(
fetch_data.select_db("stock_db", "share_base_info",
["share_id", "market_type"],
{"share_id": [ids, "in"]}))
return share_ids
def _get_ex_dividend_last_time(self, share_id, table_name):
stock_conn_manager_obj = stock_conn_manager()
conn_name = stock_conn_manager_obj.get_conn_name(share_id)
data = fetch_data.get_data(
fetch_data.select_db(conn_name,
table_name, ["time"], {},
extend="order by time desc limit 1"))
if len(data) > 0:
last_day = data[0][0]
return int(last_day)
else:
return 0
def _get_start_time(self, share_id, table_name, ma_empty_start_time):
stock_conn_manager_obj = stock_conn_manager()
conn_name = stock_conn_manager_obj.get_conn_name(share_id)
last_time = fetch_data.get_data(fetch_data.select_db(conn_name, table_name, ["time"], {"time":[ma_empty_start_time, "<="]}, extend="order by time desc limit 180"))
if len(last_time) > 0:
last_day = last_time[-1][0]
tz = pytz.timezone('Asia/Shanghai')
last_day_obj = datetime.datetime.fromtimestamp(last_day, tz)
time_str = last_day_obj.strftime("%Y%m%d")
return time.mktime(time.strptime(time_str, '%Y%m%d'))
else:
return ma_empty_start_time
def _parse_data_and_insert_db(self, table_name, data):
data_array = fetch_data.get_data(
fetch_data.regular_split("string_comma_regular", data))
db_manager = mysql_manager()
conn = db_manager.get_mysql_conn(self._conn_name)
into_db_columns = [
"market_type", "share_id", "share_name", "price", "up_percent",
"market_maker_net_inflow", "market_maker_net_inflow_per",
"huge_inflow", "huge_inflow_per", "large_inflow",
"large_inflow_per", "medium_inflow", "medium_inflow_per",
"small_inflow", "small_inflow_per", "time_str"
]
conn.insert_data(table_name, into_db_columns, data_array)
def _bak_market_maker_info(self, share_id):
date_info = time.strftime('%Y_%m_%d')
table_name = "market_maker_%s" % (date_info)
fields_array = [
"time_str", "price", "up_percent", "market_maker_net_inflow",
"market_maker_net_inflow_per", "huge_inflow", "huge_inflow_per",
"large_inflow", "large_inflow_per", "medium_inflow",
"medium_inflow_per", "small_inflow", "small_inflow_per"
]
daily_data = fetch_data.get_data(
fetch_data.select_db(self._daily_temp_conn_name, table_name,
fields_array, {"share_id": [share_id, "="]}))
self._bak_single_market_maker_info(share_id, daily_data)
def _get_close_volume(self, share_id, table_name, start_time):
stock_conn_manager_obj = stock_conn_manager()
conn_name = stock_conn_manager_obj.get_conn_name(share_id)
data = fetch_data.get_data(fetch_data.select_db(conn_name, table_name, ["time", "today_close", "volume"], {"time":[start_time, ">="]}))
time_list = []
close_list = []
volume_list = []
for item in data:
time_int = item[0]
close = item[1]
volume = item[2]
time_list.append(time_int)
close_list.append(close)
volume_list.append(volume)
return {"time":time_list, "close":close_list, "volume":volume_list}
def _filter_data(self, data):
data = data.replace("None", "0")
filter_data = fetch_data.get_data(
fetch_data.regular_split("quotes_money_163", data))
if len(filter_data) > 0:
del filter_data[0]
useful_data = []
for item in filter_data:
if int(item[-2]) == 0:
continue
time_str = item[0]
time_int = time.mktime(time.strptime(time_str, '%Y-%m-%d'))
item.insert(0, time_int)
del item[2]
del item[2]
useful_data.append(item)
return useful_data
def _get_start_time(self, share_id, table_name):
stock_conn_manager_obj = stock_conn_manager()
conn_name = stock_conn_manager_obj.get_conn_name(share_id)
last_time = fetch_data.get_data(
fetch_data.select_db(conn_name,
table_name, ["time"], {},
extend="order by time desc limit 1"))
if len(last_time) > 0:
last_day = last_time[0][0]
tz = pytz.timezone('Asia/Shanghai')
last_day_obj = datetime.datetime.fromtimestamp(last_day, tz)
while True:
next_day_obj = last_day_obj + datetime.timedelta(days=1)
if next_day_obj.weekday() < 5:
break
last_day_obj = next_day_obj
time_str = next_day_obj.strftime("%Y%m%d")
else:
time_str = "19900101"
return time.mktime(time.strptime(time_str, '%Y%m%d'))
def _get_average_info(self, share_id, table_name, period = 0):
stock_conn_manager_obj = stock_conn_manager()
conn_name = stock_conn_manager_obj.get_conn_name(share_id)
periods = [5, 10, 20, 30, 60, 120]
types = ["close_ma", "volume_ma"]
columns = ["time", "today_close", "today_high", "today_low", "today_open", "yesterday_close", "pchg", "turnover_rate", "volume", "turnover"]
for type_item in types:
for period_item in periods:
column_name = "%s%d" % (type_item, period_item)
columns.append(column_name)
extend_str = "order by time desc"
if period > 0:
extend_str = "%s limit %d" % (extend_str, period)
data= fetch_data.get_data(fetch_data.select_db(conn_name, table_name, columns, {}, extend=extend_str))
infos = []
for data_item in data:
info = {}
for index in range(len(columns)):
info[columns[index]] = data_item[index]
infos.insert(0, info)
return infos
import glob
from PIL import Image
import os
import torch
from torch.utils.data import Dataset, DataLoader
from preprocess import preprocessor, DatalisttoDataset
from fetch_data import get_data
import torch.nn as nn
import torch.nn.functional as F
import torchvision.transforms as transforms
from collections import OrderedDict
batch_size = 100
num_epoch = 50
data, label = get_data(10, num_ratio=10, domain="target", mode="choiced")
print("preprocess finished")
dataset = DatalisttoDataset(data, label, transform=None)
train_dataset, test_dataset = torch.utils.data.random_split(
dataset, [int(len(dataset) * 0.8),
len(dataset) - int(len(dataset) * 0.8)])
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)
#device = "cpu"
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 32, 3)
self.conv2 = nn.Conv2d(32, 32, 3)
def get_songs(): for i in range(num): mc.add_string(get_data())
def _get_data(self, ids):
url_pre = "http://hq.sinajs.cn/list="
url = url_pre + ids
res = fetch_data.get_data(fetch_data.query_http(url))
res = res.decode("gbk").encode("utf-8")
return res
import glob
from PIL import Image
import os
import torch
from torch.utils.data import Dataset, DataLoader
from preprocess import preprocessor, DatalisttoDataset
from fetch_data import get_data
import torch.nn as nn
import torch.nn.functional as F
import torchvision.transforms as transforms
from collections import OrderedDict
from models import Net_google
batch_size = 100
num_epoch = 50
data, label = get_data(10, num_ratio = 10, domain = "target", mode = "choiced", data_aug = True)
print("preprocess finished")
dataset = DatalisttoDataset(data, label, transform = None)
#train_dataset, test_dataset = torch.utils.data.random_split(dataset, [int(len(dataset)*0.8), len(dataset) - int(len(dataset)*0.8)])
train_dataset = dataset
data_te, label_te = get_data(10, num_ratio = 10, domain = "target", mode = "processed", data_aug = False)
test_dataset = DatalisttoDataset(data_te, label_te)
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)
#device = "cpu"
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
net = Net_google().to(device)
model = Net_google()
#転移学習するときはここを使う
import os
import torch
from torch.utils.data import Dataset, DataLoader
from preprocess import DatalisttoDataset
import torch.nn as nn
import torch.nn.functional as F
import torchvision.transforms as transforms
from collections import OrderedDict
from models import Net_google
from fetch_data import get_data
batch_size = 100
num_epoch = 50
data, label = get_data(10,
num_ratio=10,
domain="source",
mode="processed",
data_aug=False)
print("preprocess finished")
dataset = DatalisttoDataset(data, label, transform=None)
train_dataset, test_dataset = torch.utils.data.random_split(
dataset, [int(len(dataset) * 0.8),
len(dataset) - int(len(dataset) * 0.8)])
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)
device = "cpu"
#device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = Net_google()
model.load_state_dict(torch.load("net.model"))
#checkpoint = torch.load("net.model")
def _get_data(self, market_type, id, start_time, end_time):
url_format = """http://quotes.money.163.com/service/chddata.html?code=%d%s&start=%s&end=%s&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;PCHG;TURNOVER;VOTURNOVER;VATURNOVER"""
url = url_format % (market_type, id, start_time, end_time)
res = fetch_data.get_data(fetch_data.query_http(url))
#res = res.decode("gbk").encode("utf-8")
return res
def get_wards():
return fetch_data.get_data()
def _get_data(self):
date_info = time.strftime('%Y-%m-%d')
url_fomart = "http://nufm.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx/JS.aspx?type=ct&st=(BalFlowMain)&sr=-1&p=3&ps=%d&js=var%%20vDUaFOen={pages:(pc),date:%%22%s%%22,data:[(x)]}&token=894050c76af8597a853f5b408b759f5d&cmd=C._AB&sty=DCFFITA&rt=49430148"
url = format(url_fomart % (3500, date_info))
res = fetch_data.get_data(fetch_data.query_http(url))
return res
def _get_data(self):
url = r"http://nufm.dfcfw.com/EM_Finance2014NumericApplication/JS.aspx/JS.aspx?type=ct&st=(FFRank)&sr=1&p=1&ps=3500&js=var%20mozselQI={pages:(pc),data:[(x)]}&token=894050c76af8597a853f5b408b759f5d&cmd=C._AB&sty=DCFFITAM&rt=49461817"
res = fetch_data.get_data(fetch_data.query_http(url))
return res
