def start_collecting():
db = Data()
while True:
result_dict = {}
_from = time.time()
result_dict['time'] = time.strftime("%Y.%m.%d - %H:%M:%S")
cur_setup = db.get_cur_setup()
list_ = []
for key, item in cur_setup.items():
list_.append([key, item])
pool = ThreadPool(5)
ret = pool.map(do_func, list_)
pool.terminate()
pool.join()
for jdx, item in enumerate(list_):
tmp_dict = {}
for idx, pair in enumerate(item[1]):
tmp_dict[pair] = ret[jdx][idx]
result_dict[item[0]] = tmp_dict
#print (result_dict)
db.insert_tick(result_dict)
end_time = time.time() - _from
if end_time < 1:
time.sleep(1 - end_time)
def imp_delta_plot(self):
sql = r"""
select t1.delta, t2.delta from tb_sec_delta t1 inner join tb_sec_delta t2
on t1.code = t2.code and t2.seq = t1.seq + %s
where t1.seq = 0 and t1.delta is not null and t2.delta is not null
and t1.delta between -30 and 20
"""
sql0 = r"select delta from tb_sec_delta where seq = 0 and delta is not null"
imp = np.array(Data(sql0, self.cur).select_col(0))
fig = plt.figure(figsize=(7.2, 9.6))
plt.subplot(321)
plt.hist(imp, bins=np.arange(-40, 30, 1), normed=True)
plt.title("一级发行冲击分布图", fontproperties="SimHei")
plt.xlabel("发行冲击（bp)", fontproperties="SimHei")
for i in range(2, 7):
data = np.array(Data(sql, self.cur, (i - 1, )).data)
eval("plt.subplot(32{})".format(i))
X = sm.add_constant(data[:, 0])
Y = data[:, 1]
model = sm.OLS(Y, X)
res = model.fit()
formula_string = "y = {:.4f} + {:.4f}*x".format(*res.params)
line_x = [-30, 20]
line_y = res.predict(sm.add_constant(line_x))
plt.scatter(data[:, 0], data[:, 1], s=0.5)
plt.plot(line_x, line_y, color='black')
plt.text(line_x[0], 6, formula_string)
plt.title("发行第{}天的收益变动散点图".format(i - 1), fontproperties="SimHei")
plt.xlabel("发行冲击（bp）", fontproperties="SimHei")
plt.ylabel("二级市场收益率变动（bp)", fontproperties="SimHei")
plt.xlim(*line_x)
plt.ylim(-7, 7)
plt.show()
def build(cls):
#if exists(cls.pickle_fn()):
# result = pickle.load(open(cls.pickle_fn(), "rb" ) )
tv = TreeView(hide_root=True)
tv.size_hint = 1, None
tv.bind(minimum_height=tv.setter('height'))
data = Data()
groups = data.get_groups()
ingredients = data.get_ingredients()
def already_created(node, text):
if hasattr(node, 'text'):
return node.text == text
else:
return False
for group in groups:
if len(
list(
filter(
lambda seq: already_created(seq, group['group']),
tv.iterate_all_nodes()))) == 0:
node_group = tv.add_node(TreeViewLabel(text=group['group']))
node_group = list(
filter(lambda seq: already_created(seq, group['group']),
tv.iterate_all_nodes()))
if len(node_group) > 0:
if len(
list(
filter(
lambda seq: already_created(
seq, group['subgroup']),
tv.iterate_all_nodes()))) == 0:
node_subgroup = tv.add_node(
TreeViewLabel(text=group['subgroup']), node_group[0])
for ingredient in ingredients:
node_subgroup = list(
filter(
lambda seq: already_created(seq, ingredient['food_subgroup'
]),
tv.iterate_all_nodes()))
if len(node_subgroup) > 0:
tv.add_node(
IngredientListPopupItem(
prop_id=ingredient['id'],
name=ingredient['name'],
name_scientific=ingredient['name_scientific'],
description=ingredient['description']),
node_subgroup[0])
else:
print('error adding {0}', ingredient['name'])
cls.tv = tv
def __init__(self):
self.settings = Settings()
self.data = Data()
country_string = self.settings.get_value(Settings.COUNTRIES_MODEL)
countries = country_string.split(",")
self.xs = []
self.ys = []
# points to the end of the last batch
self.train_batch_pointer = 0
self.val_batch_pointer = 0
# Get all images
self.image_list = []
# Workaround
self.image_list = self.data.get_image_list_filter(maneuver=0)
#for country in countries:
# self.image_list += self.data.get_image_list_filter(country=country, maneuver=0)
for image in self.image_list:
self.steering_deg = float(image[2]) * scipy.pi / 180
# higher steering angles are rare, so add four times
# if abs(steering_deg) > 40:
# for i in range(int(steering_deg/10-2)*4):
# xs.append("../captured/" + line.split()[0])
# ys.append(steering_deg)
self.xs.append(os.path.join("captured/", image[1]))
# the paper by Nvidia uses the inverse of the turning radius,
# but steering wheel angle is proportional to the inverse of turning radius
# so the steering wheel angle in radians is used as the output
self.ys.append(self.steering_deg)
# get number of images
self.num_images = len(self.xs)
# shuffle list of images
self.c = list(zip(self.xs, self.ys))
random.shuffle(self.c)
self.xs, self.ys = zip(*self.c)
# Training data
self.train_xs = self.xs[:int(len(self.xs) * 0.8)]
self.train_ys = self.ys[:int(len(self.xs) * 0.8)]
# Validation data
self.val_xs = self.xs[-int(len(self.xs) * 0.2):]
self.val_ys = self.ys[-int(len(self.xs) * 0.2):]
self.num_train_images = len(self.train_xs)
self.num_val_images = len(self.val_xs)
print("Total data:", len(self.xs), self.num_images)
print("Training data:", len(self.train_xs))
print("Validation data:", len(self.val_xs))
def newsubmit():
form = SubmitForm()
if form.validate_on_submit():
submitted = Data()
submitted.c4cid = random.random()
form.populate_obj(submitted)
result = getClassify(submitted.c4cid, submitted.message)
print(result)
return redirect(url_for('index'))
def imp_minutes(self, bond_type, future_type, delta_type, day):
"""计算发行冲击当日的五分钟级的市场走势"""
if future_type == "TF":
future_term = 5
elif future_type == "T":
future_term = 10
else:
raise ValueError("不被接受的参数值future_type")
# 获得delta五等分点
sql1 = """select t1.{} from impact t1 inner join future_minute t2
on t1.dt = date(t2.dtt) and t2.seq=0
where t1.bondtype = %s and t2.term = %s
""".format(delta_type)
delta = np.array(Data(sql1, self.cur, (bond_type, future_term)).data)
delta = pd.DataFrame(delta, columns=["delta"]).dropna()
per_delta = [float(delta.min() - 1)]
for p in range(20, 120, 20):
per_delta.append(float(np.percentile(delta, p)))
# 根据五等分点（per_delta)从数据库中选出每个分位的
data = []
if day == 0:
sql2 = r"""select date_format(t2.dtt, '%%H:%%i'), avg(t2.close) from impact t1
inner join future_minute t2
on t1.dt = date(t2.dtt) and t1.bondtype = %s and t2.term = %s
and t1.{0} > %s and t1.{0} <= %s
group by date_format(t2.dtt, '%%H:%%i')
""".format(delta_type)
elif day > 0:
sql2 = r"""select date_format(t2.dtt, '%%H:%%i'), avg(t2.close) from impact t1
inner join future_minute t2 inner join dts1 t3 inner join dts1 t4
on t1.dt = t3.dt and t4.seq = t3.seq +{0} and t4.dt = date(t2.dtt)
where t1.bondtype = %s and t2.term = %s and t1.{1} > %s and t1.{1} <= %s
group by date_format(t2.dtt, '%%H:%%i')
""".format(day, delta_type)
else:
sql2 = r"""select date_format(t2.dtt, '%%H:%%i'), avg(t2.close) from impact t1
inner join future_minute t2 inner join dts1 t3 inner join dts1 t4
on t1.dt = t3.dt and t4.seq = t3.seq - {0} and t4.dt = date(t2.dtt)
where t1.bondtype = %s and t2.term = %s and t1.{1} > %s and t1.{1} <= %s
group by date_format(t2.dtt, '%%H:%%i')
""".format(abs(day), delta_type)
for i in range(len(per_delta) - 1):
a = per_delta[i]
b = per_delta[i + 1]
da = Data(sql2, self.cur, (bond_type, future_term, a, b))
time_index = da.select_col(0)
rate = da.select_col(1)
data.append(rate)
data = np.array(data).T
res = []
for k in range(1, len(data), 1):
res.append(100 * (data[k] - data[0]))
res = pd.DataFrame(res,
index=time_index[1:],
columns=["一", "二", "三", "四", "五"])
return res
def delete_selection(self):
img_id = self._get_selected_image()
if not img_id:
return
filename = Data().get_image_data(img_id)[1]
Data().delete_image(filename)
if os.path.exists(os.path.join("captured", filename)):
os.remove(os.path.join("captured", filename))
self.fill_image_list()
def test_persist_classified_with_new_records_returns_correct_diff_record_count(
self):
data = Data(self.db_name)
handler = FileHandler()
seed = handler.read_seed_csv(self.seed_file)
data.seed_transactions(seed)
new_data = handler.read_classified_csv(self.classified)
result = data.persist_classified(new_data)
self.assertEqual(144, result)
def fill_sequence_data_fields(self):
"""
Set text of e_country and choose right cb_roadtype.
"""
sid = self._get_selected_sequence()
if sid:
sequence_data = Data().get_sequence_data(sid)
if sequence_data:
code = Data().get_country_code(sequence_data[1])
self.ui.e_country.setText(code)
self.ui.cb_roadtype.setCurrentIndex(sequence_data[2] + 1)
def imp_days_minutes(self,
day1,
day2,
future_type,
bond_type="国债",
delta_type="delta",
k=5,
p2r_mode=0):
"""计算发行前day1日至发行后day2日的国债期货五分钟行情序列，k为等分数，默认为5等分"""
if future_type == "TF":
future_term = 5
elif future_type == "T":
future_term = 10
else:
raise ValueError("不被接受的参数值future_type")
dt0 = dtt.date(2016, 1, 8)
dt1 = dt0 + dtt.timedelta(days=day1)
sdt = dt1.strftime("%Y-%m-%d")
sql1 = """
select t1.dt, t1.term, t1.{0}, t2.dt, t3.dt, t1.bondtype
from impact t1 inner join dts1 t2 inner join dts1 t3 inner join dts1 t4
on t1.dt = t4.dt and t2.seq = t4.seq - %s and t3.seq = t4.seq + %s and t1.bondtype = "{1}"
where t1.{0} is not null and t1.dt >= '{2}'
order by t1.{0}
""".format(delta_type, bond_type, sdt)
data1 = Data(sql1, self.cur, (day1, day2)).data
num = len(data1) # 提取记录的个数，用于
print(num)
# 提取交易行情序列
data2 = []
sql2 = """
select close from future_minute
where date(dtt) between %s and %s and term = {}
""".format(future_term)
for d1 in data1:
d2 = Data(sql2, self.cur, (d1[3], d1[4])).select_col(0)
d2 = p2r(d2, mode=p2r_mode)
data2.append(d2)
n = round(num / k) # 每个分位的记录个数
res = [] # 结果res用于保存
for i in range(k):
a = i * n
if i == k - 1:
b = num
else:
b = n * (i + 1)
r = np.mean(data2[a:b], axis=0)
res.append(r)
return res
def imp_days(self, bond_type, future_type):
"""将发行冲击五等分，计算之后4日的国债期货收益均值，参数bond_type为续发债类型，分别为
国债和国开债，future_type为国债期货合约类型("TF"或者"T")"""
if future_type == "TF":
future_term = 5
elif future_type == "T":
future_term = 10
else:
raise ValueError("不被接受的参数值future_term")
sql1 = """select t1.mg_delta, t2.dclose, t3.dclose, t4.dclose, t5.dclose, t6.dclose, t7.dclose, t8.dclose
from impact t1 inner join future_delta t2 inner join future_delta t3 inner join future_delta t4
inner join future_delta t5 inner join future_delta t6 inner join future_delta t7 inner join future_delta t8
on t1.dt = t5.dt and t2.seq = t5.seq - 3 and t3.seq = t5.seq - 2 and t4.seq = t5.seq - 1
and t6.seq = t5.seq+1 and t7.seq = t5.seq+2 and t8.seq = t5.seq +3
and t2.term = t5.term and t3.term = t5.term and t4.term = t5.term and t6.term = t5.term
and t7.term = t5.term and t8.term = t5.term
and t1.bondtype = %s and t5.term = %s
order by t1.delta
"""
data = Data(sql1, self.cur, (bond_type, future_term)).data
data = pd.DataFrame(np.array(data))
# 依据delta将data五等分
n = 5
res = []
l = int(len(data) / n)
for i in range(n):
a = i * l
b = (i + 1) * l - 1
if i == 4:
b = -1
d = list(data[a:b].mean())
d.insert(1, len(data[a:b]))
res.append(d)
return res
class DockerExecutor():
#Instantiating the docker client.
def __init__(self, parsed_data):
self.pack_info = parsed_data
self.db =Data(constants.PACKAGE_TABLE_NAME)
self.package = self.db.find(self.pack_info["id"])
self.container = None
try:
self.docker_cli = docker.Client(base_url="tcp://192.168.99.100:2375",
version="1.18",
timeout=600)
except:
print "Docker connection Error."
print "Setting the Client for docker...."
#Creating the container for executing the test suites.
def create_container(self):
try:
print "Creating the container...."
self.container = self.docker_cli.create_container(image="pyhold:1.0",
tty=True)
except:
print "Error creating & starting the container."
#Starting the container
def start_container(self):
try:
print "Starting the container...."
self.docker_cli.start(self.container)
except Exception,err:
print Exception, err
print "Error while starting container."
def imp_select_code(imp: list, cur, column="delta"):
res = list()
res.append(
Data(
r"select code from tb_sec_delta where seq=0 and {} <= %s".format(
column), cur, (imp[0], )).select_col(0))
for i in range(len(imp) - 1):
res.append(
Data(
r"select code from tb_sec_delta where seq=0 and {} > %s and delta <= %s"
.format(column), cur, (imp[i], imp[i + 1])).select_col(0))
res.append(
Data(
r"select code from tb_sec_delta where seq=0 and {} > %s".format(
column), cur, (imp[-1], )).select_col(0))
return res
def get_last_position_value(self, cash, ps, dt):
"""对于持仓最后一天的持仓市值的计算，get_position_price无能为力，因此特别编制本方法来处理"""
symbols = ps.keys()
if symbols:
price_list = []
sql = r"""select distinct dirty from tb_sec where dt = %s and code0 = %s"""
for symbol in symbols:
price_list.append(
Data(sql, self.cur, (dt, symbol)).select_col(0))
if price_list:
prices = np.matrix(price_list, dtype=float)
else:
raise ValueError("未从数据库中查找到所需价格信息，日期：{}、品种：{}".format(
dt, price_list))
volumes = np.matrix(list(ps.values()), dtype=float)
asset_matrix = np.dot(volumes, prices)
cash_matrix = np.matrix(cash)
total_matrix = asset_matrix + cash_matrix
data_matrix = np.concatenate(
(cash_matrix, asset_matrix, total_matrix), axis=1)
return pd.DataFrame(data_matrix,
index=pd.to_datetime([dt]),
columns=["cash", "asset", "total"])
else:
return pd.DataFrame([[cash, 0, cash]],
index=pd.to_datetime([dt]),
columns=["cash", "asset", "total"])
def fill_image_list(self):
"""
Fill list_image with data.
"""
data = Data()
list_images = self.ui.list_images
model = QStandardItemModel(list_images)
images = data.get_image_list(self.sequence_id)
if len(images) > 0:
for image in images:
item = QStandardItem(
"%s - %s" % (image[1], functions.get_indicator(image[5])))
item.setEditable(False)
item.setData(str(image[0]), QtCore.Qt.UserRole)
model.appendRow(item)
list_images.setModel(model)
class Controller:
def __init__(self, classifier, db_name, class_csv_name):
self.data = Data(db_name)
self.handler = FileHandler()
self.classifier = classifier
self.class_csv = class_csv_name
def get_new_transactions(self, file_name):
df = self.handler.read_csv(file_name)
return self.data.persist_unclassified(df)
def seed_database(self, file_name):
df = self.handler.read_seed_csv(file_name)
return self.data.seed_transactions(df)
def classify_new_transactions(self):
self.__train_model()
classified = self.__classify_new()
if classified is not None:
self.__persist_prediction(classified)
def all_to_csv(self, file_name):
self.data.get_classified().to_csv(file_name)
def __train_model(self):
train_data = self.data.get_classified()
self.classifier.train(train_data)
def __classify_new(self):
unclassified = self.data.get_unclassified()
if unclassified.shape[0] <= 0:
return None
return self.classifier.classify(unclassified)
# TODO this module cleans up merge from
def __persist_prediction(self, classified):
self.data.persist_prediction(classified)
self.handler.write_csv(classified, self.class_csv)
def confirm_csv_classification(self):
df = self.handler.read_classified_csv(self.class_csv)
return self.data.persist_classified(df)
def get_nett_balance(self, start, end):
df = self.data.get_range("transactions", start, end)
return sum(df["Amount"].astype(float) * df["Labels"].astype(int))
def get_position_price(self, ps: dict, dt1, dt2):
"""根据持仓从数据库中提取相应的价格数据,以矩阵形式返回结果，行表示品种，列表示日期，参数ps表示字典，键为品种，值
为持仓量，dt1表示开始日期，dt2表示结束日期，价格序列包含开始日期而不包含结束日期"""
symbols = ps.keys()
price_list = []
sql = r"""select distinct dirty from tb_sec where dt >= %s and dt < %s and code0 = %s order by dt"""
for symbol in symbols:
price_list.append(
Data(sql, self.cur, (dt1, dt2, symbol)).select_col(0))
if price_list:
prices = np.matrix(price_list, dtype=float)
else:
prices = None
dts = Data(
"select distinct dt from tb_sec where dt >= %s and dt < %s order by dt",
self.cur, (dt1, dt2)).select_col(0)
return prices, dts
def delete_selected_sequence(self):
"""
b_sequenceDelete:
Delete the selected sequence.
"""
sid = self._get_selected_sequence()
Data().delete_sequence(sid)
self.fill_sequence_list()
def migrate():
data = Data(batch=True)
sequences = []
# read sequence.txt
with open("captured/sequence.txt") as f:
for line in f:
split = line.split()
# Start, End, Country, Type
timestamp = datetime.datetime.fromtimestamp(
round(
os.path.getctime(
os.path.join("captured", "%s.png" %
split[0])))).strftime("%Y-%m-%d %H:%M:%S")
sid = data.add_sequence(split[2],
int(split[3]),
timestamp=timestamp)
sequences.append([int(split[0]), int(split[1]), sid])
# read data.txt
num_datasets = sum(1 for line in open("captured/data.txt"))
datasets_writen = 0
percent = round(num_datasets / 100)
with open("captured/data.txt") as f:
for line in f:
datasets_writen += 1
split = line.split()
sid = get_sid(sequences, int(split[0].split(".png")[0]))
data.add_image(split[0], split[1], split[2], split[3], split[4],
sid)
if datasets_writen % percent == 0:
print("%d %%" % round(datasets_writen / num_datasets * 100))
data.append()
def append_sequence_changes(self):
"""
b_sequenceApply:
Update data of selected sequence.
"""
sid = self._get_selected_sequence()
code = self.ui.e_country.text()
road_type = self.ui.cb_roadtype.currentIndex() - 1
Data().update_sequence(sid, code, road_type)
def get_avg_std_by_term(self, terms, column="delta"):
"""计算不同期限的发行冲击的均值与标准差"""
res = []
sql = "select count(*), avg({0}), stddev_samp({0}) from tb_sec_delta " \
"where seq = 0 and term = %s".format(column)
for t in terms:
num, avg, std = Data(sql, self.cur, (t, )).data[0]
res.append([num, avg, std])
return res
def fill_sequence_list(self):
"""
Fill sequence_list with data.
"""
data = Data()
sequence_list = self.ui.sequence_list
model = QStandardItemModel(sequence_list)
sequences = data.get_sequence_list()
if len(sequences) > 0:
for sequence in sequences:
note = ""
if sequence[4] is not None:
note = " - %s" % sequence[4]
item = QStandardItem("%s%s" % (sequence[1], note))
item.setEditable(False)
item.setData(str(sequence[0]), QtCore.Qt.UserRole)
model.insertRow(0, item)
sequence_list.setModel(model)
async def handler(self, conn, loop):
obj=Data()
hash=SHA256.new()
username=""
password=""
public_key=""
await loop.sock_sendall(conn, str.encode("Enter username: "******"utf-8")
if res:
username=res
await loop.sock_sendall(conn, str.encode("Enter password: "******"users", username, password)
if not result:
obj.user_entry(username, password)
await loop.sock_sendall(conn, str.encode("Enter public key: "))
res_en=await loop.sock_recv(conn, 2048)
res=res_en.decode("utf-8")
if res:
try:
public_key=eval(res)
except Exception as e:
print(str(e))
await loop.sock_sendall(conn, str.encode("Name a group that you would want to create or join: "))
res_en=await loop.sock_recv(conn, 2048)
res=res_en.decode("utf-8")
if res is not None:
obj.create_table(res)
key=random.randint(100000, 1000000)
key=self.crypt.encrypt(str(key), public_key)
await loop.sock_sendall(conn, str.encode(key))
result=obj.check_table(res, username, password)
if not result:
obj.group_entry(res, username, password, key)
conn.close()
def __init__(self, parsed_data):
self.pack_info = parsed_data
self.db =Data(constants.PACKAGE_TABLE_NAME)
self.package = self.db.find(self.pack_info["id"])
self.container = None
try:
self.docker_cli = docker.Client(base_url="tcp://192.168.99.100:2375",
version="1.18",
timeout=600)
except:
print "Docker connection Error."
print "Setting the Client for docker...."
def imp_seq(self, imp: list, seq: list, column="delta"):
"""返回根据冲击大小与seq先后排序的二维收益率差"""
res = list()
sql = r"""select avg({}) from tb_sec_delta where code in %s and seq = %s""".format(
column)
codes = imp_select_code(imp, self.cur, column=column)
for code in codes:
num = len(code)
d = list([num])
for s in seq:
d.append(Data(sql, self.cur, (code, s)).data[0][0])
res.append(d)
return res
def imp_future(self, imp: list, seq: list, term=10, delta="dsrate"):
""""发行冲击对国债期货市场影响"""
res = list()
sql0 = """
select count(*), avg(t1.delta) from tb_sec_delta t1 inner join future_delta t2
on t1.dt = t2.dt
where t1.code in %s and t2.term = %s and t1.seq = 0
"""
sql = """
select avg(t2.{})
from tb_sec_delta t1 left outer join future_delta t2
on t1.dt = t2.dt
where t1.code in %s and t2.term = %s and t1.seq = %s
""".format(delta)
codes = imp_select_code(imp, self.cur)
for code in codes:
data_imp = Data(sql0, self.cur, (code, term)).data
num = data_imp[0][0]
avg_imp = data_imp[0][1]
d = list([num, avg_imp])
for s in seq:
d.append(Data(sql, self.cur, (code, term, s)).data[0][0])
res.append(d)
return res
def success():
if request.method == 'POST':
action = request.form["action"]
# Adding a new data into database
if action == "add":
name = request.form["Project_name"]
programming_language = request.form["Programming_Language"]
description = request.form["description"]
git_url = request.form["git_url"]
request_data = Data(name, programming_language, description,
git_url)
db.session.add(request_data)
db.session.commit()
return render_template("success.html")
# Modify already exist database
elif action == "modify":
id = request.form["id"]
name = request.form["Project_name"]
programming_language = request.form["Programming_Language"]
description = request.form["description"]
git_url = request.form["git_url"]
modify_object = Data.query.filter_by(id=id).one()
if len(name) != 0:
modify_object.name = name
if len(programming_language) != 0:
modify_object.programming_language = programming_language
if len(description) != 0:
modify_object.description = description
if len(git_url) != 0:
modify_object.git_url = git_url
db.session.commit()
return render_template("success.html")
# Delete an instance from data
elif action == "delete":
id = request.form["id"]
delete_object = Data.query.filter_by(id=id).one()
db.session.delete(delete_object)
db.session.commit()
return render_template("success.html")
def listen(ch, method, properties, body):
self.datasql = Data(constants.PACKAGE_TABLE_NAME)
self.rwriter = BuildLogWriter()
print colored("[x] Received -- ", 'magenta') , body
self.data = body
self.parsed_data = json.loads(self.data)
print self.parsed_data
self.pack = self.datasql.find(self.parsed_data["id"])
print "PACKAGE => ",self.pack
self.file_name = str(self.datasql.find(self.parsed_data["id"],select="file_name")[0][0])
self.lib_name = str(self.datasql.find(self.parsed_data["id"],select="lib_name")[0][0])
dock = DockerExecutor(self.parsed_data)
dock.create_container()
dock.start_container()
print dock.exec_command("mkdir testex")
package_endpoint = constants.PACKAGE_ENDPT + str(self.file_name)
print dock.exec_command('bash -c "cd testex && wget %s"' % str(package_endpoint))
print dock.exec_command('bash -c "cd testex && tar -xvf %s"' % self.file_name)
#Requirements Installation
self.req_data = dock.exec_command('bash -c "cd / && cd testex && cd %s && pip install -r requirements.txt"' % self.lib_name)
print self.req_data
self.rwriter.write(self.lib_name, "requirements_installation.log", self.req_data)
#Custom build commands
self.build_commands = self.datasql.find(self.parsed_data["id"],select="commands")
if(not(self.build_commands[0][0] == "")):
print "BUILD COMMANDS => ", self.build_commands[0][0]
commands = self.build_commands[0][0].split(",")
for com in enumerate(commands):
self.req_data = dock.exec_command('bash -c "cd / && cd testex && cd %s && %s"' % (self.lib_name, com[1]))
print self.req_data
self.rwriter.write(self.lib_name,"custom_build_" + str(com[0]) + ".log", self.req_data)
#Building from source
self.build_data = dock.exec_command('bash -c "cd / && cd testex && cd %s && python setup.py install"' % self.lib_name)
print self.build_data
self.rwriter.write(self.lib_name, "build.log", self.build_data)
#Stopping the container
dock.stop_container()
def imp_hist_plot(self):
"""绘制发行冲击的直方分布图，分别使用收益率变动与净价变动来衡量"""
fig, axes = plt.subplots(2, 2, figsize=(9.6, 4.8))
fig.subplots_adjust(hspace=0.5)
i = 0
for bondtype in ["00", "02"]:
j = 0
for column in ["delta", "dprice"]:
sql = r"""select {0} from tb_sec_delta where seq = 0 and {0} is not null and
code0 regexp '[:alnum:]{{2}}{1}.*'""".format(
column, bondtype)
imp = np.array(Data(sql, self.cur).select_col(0))
if column == "delta":
if bondtype == "00":
bins = np.arange(-40, 30, 1)
title = "国债发行冲击(BP)分布图"
elif bondtype == "02":
bins = np.arange(-40, 30, 1)
title = "国开债发行冲击(BP)分布图"
else:
raise ValueError("错误的参数值bondtype")
elif column == "dprice":
if bondtype == "02":
bins = np.arange(-2, 4, 0.01)
title = "国开债发行冲击(元)分布图"
elif bondtype == "00":
bins = np.arange(-2, 4, 0.01)
title = "国债发行冲击(元)分布图"
else:
raise ValueError("错误的参数值bondtype")
else:
raise ValueError("错误的参数值column")
axes[i, j].hist(imp, bins=bins)
axes[i, j].set_title(title,
fontproperties="SimHei",
fontsize=20)
j += 1
i += 1
fig.show()
def get_paymentdt(self, codes, time1, time2):
"""根据持仓债券代码查询到期日期，以及付息额,如果存在付息情况，则返回结果中包含债券代码与付息额"""
res = []
if codes:
sql = r"select * from payment where code in %s"
data = Data(sql, self.cur, (codes, )).data
for d in data:
if time1.year == time2.year:
dt = dtt.date(time1.year, d[1].month, d[1].day)
if time1 < dt <= time2:
res.append([d[0], dt, d[2]])
elif time1.year < time2.year:
delta_year = time2.year - time1.year
if delta_year == 1:
if d[1].month > time1.month:
dt = dtt.date(time1.year, d[1].month, d[1].day)
res.append([d[0], dt, d[2]])
if d[1].month < time2.month:
dt = dtt.date(time2.year, d[1].month, d[1].day)
res.append([d[0], dt, d[2]])
elif d[1].month < time2.month:
dt = dtt.date(time2.year, d[1].month, d[1].day)
res.append([d[0], dt, d[2]])
elif d[1].month == time1.month:
if d[1].day > time1.day:
dt = dtt.date(time1.year, d[1].month, d[1].day)
res.append([d[0], dt, d[2]])
elif d[1].month == time2.month:
if d[1].day <= time2.day:
dt = dtt.date(time2.year, d[1].month, d[1].day)
res.append([d[0], dt, d[2]])
else:
pass # delta_year的年限超过1年已超出本项目的需要，因此虽然有考虑这种情况但是不做处理
else:
raise ValueError("time1应当小于time2")
return res
def test_fit_model(self):
with self.assertRaises(RegressionException):
self._data.fit_model(1, Data('ideal'), 'real')
self.df = pd.DataFrame(
[[1.5555555, 2.55555555], [4.5555555, 5.5555555]],
columns=['x', 'y1'])
self.df.to_csv('./tests/unittest.csv')
self._data.csv_to_df()
with self.assertRaises(Exception):
self._data.fit_model(1, None, 'linear')
# test print_table arg
with open('./datasets/ideal.csv', 'r') as idf:
ideal = Data('ideal', _create=False)
ideal.csv_to_df()
self.assertFalse(ideal.is_empty(), 'df obj should be populated')
model = self._data.fit_model(1,
ideal,
'linear',
print_table=True,
table_name='./tests/unittest')
self.assertEqual(type(model),
type(Model(self.df['x'], self.df['y1'], 1)))
self.assertEqual(model.x[0], self.df['x'][0])
class MainExecutor:
def main_loop(self):
def listen(ch, method, properties, body):
self.datasql = Data(constants.PACKAGE_TABLE_NAME)
self.rwriter = BuildLogWriter()
print colored("[x] Received -- ", 'magenta') , body
self.data = body
self.parsed_data = json.loads(self.data)
print self.parsed_data
self.pack = self.datasql.find(self.parsed_data["id"])
print "PACKAGE => ",self.pack
self.file_name = str(self.datasql.find(self.parsed_data["id"],select="file_name")[0][0])
self.lib_name = str(self.datasql.find(self.parsed_data["id"],select="lib_name")[0][0])
dock = DockerExecutor(self.parsed_data)
dock.create_container()
dock.start_container()
print dock.exec_command("mkdir testex")
package_endpoint = constants.PACKAGE_ENDPT + str(self.file_name)
print dock.exec_command('bash -c "cd testex && wget %s"' % str(package_endpoint))
print dock.exec_command('bash -c "cd testex && tar -xvf %s"' % self.file_name)
#Requirements Installation
self.req_data = dock.exec_command('bash -c "cd / && cd testex && cd %s && pip install -r requirements.txt"' % self.lib_name)
print self.req_data
self.rwriter.write(self.lib_name, "requirements_installation.log", self.req_data)
#Custom build commands
self.build_commands = self.datasql.find(self.parsed_data["id"],select="commands")
if(not(self.build_commands[0][0] == "")):
print "BUILD COMMANDS => ", self.build_commands[0][0]
commands = self.build_commands[0][0].split(",")
for com in enumerate(commands):
self.req_data = dock.exec_command('bash -c "cd / && cd testex && cd %s && %s"' % (self.lib_name, com[1]))
print self.req_data
self.rwriter.write(self.lib_name,"custom_build_" + str(com[0]) + ".log", self.req_data)
#Building from source
self.build_data = dock.exec_command('bash -c "cd / && cd testex && cd %s && python setup.py install"' % self.lib_name)
print self.build_data
self.rwriter.write(self.lib_name, "build.log", self.build_data)
#Stopping the container
dock.stop_container()
try:
config = assets.Asset()
credentials = config.get_rabbit_credentials()
rabbit_credential = pika.credentials.PlainCredentials(credentials["user_name"],
credentials["password"])
connection = pika.BlockingConnection(pika.ConnectionParameters(
host=constants.PYHOLD_HOST,credentials=rabbit_credential))
self.channel = connection.channel()
self.channel.queue_declare(queue='pyhold_mq_stream')
self.channel.basic_consume(listen,
queue='pyhold_mq_stream',
no_ack=True)
od.oline()
print ">> Starting Rabbit (AMQP) Listener.", "\t", colored("[SUCCESS]","green")
print ">> Listening..."
od.oline()
self.channel.start_consuming()
except:
print colored("Error while binding/connecting to Rabbit Server / "
"Keyboard Interrupt.", "red")
print colored("Please ensure that Rabbit Server is running", "red")
def run(self):
# Settings instance
s = Settings()
# State of autopilot
autopilot = False
# Previous state of the autopilot button
autopilot_button_prev = 0
# Previous value of steering (gamepad)
manual_steering_prev = 0
img_wheel = cv2.imread('steering_wheel_image.jpg', 0)
rows, cols = img_wheel.shape
while AutopilotThread.running:
pygame.event.pump()
# Button to activate/deactivate autopilot
autopilot_button_act = self.joystick.get_button(self.b_autopilot)
# Button was pressed
if autopilot_button_act != autopilot_button_prev and autopilot_button_act == 1:
autopilot = not autopilot
#if autopilot and settings.AUTOPILOT_SOUND_ACTIVATE:
# autopilot_engage.play()
autopilot_button_prev = autopilot_button_act
# Read the steering value of joystick
axis = round(
(self.joystick.get_axis(self.steering_axis) + 1) * 32768 / 2)
# Interrupt autopilot if manual steering was detected
if abs(manual_steering_prev - axis) > 500 and autopilot:
img_id = Data().get_next_fileid()
sequence_id = Data().add_sequence(country=Settings().get_value(
Settings.COUNTRY_DEFAULT),
note="correction")
self.controller_thread.set_autopilot(False)
self.statusbar.showMessage("Autopilot inactive")
# TODO: Deactivate this feature in settings
# TODO: Amount of images to save in settings
# Save the next 3 images
for i in range(3):
# Get frame of game
frame_raw = ImageGrab.grab(
bbox=functions.get_screen_bbox())
frame = np.uint8(frame_raw)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# Relevant image region for steering angle prediction
main = frame[
s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s.
get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM),
s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s.
get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)]
# Resize image to save some space (height = 100px)
ratio = main.shape[1] / main.shape[0]
resized = cv2.resize(main, (round(ratio * 100), 100))
axis = self.joystick.get_axis(
s.get_value(Settings.STEERING_AXIS)) * 180
cv2.imwrite("captured/%d.png" % img_id, resized)
Data().add_image("%d.png" % img_id, axis, 0, 0, 0,
sequence_id)
img_id += 1
time.sleep(0.150)
autopilot = False
manual_steering_prev = axis
self.controller_thread.set_autopilot(autopilot)
# Get frame of game
frame_raw = ImageGrab.grab(bbox=functions.get_screen_bbox())
frame = np.uint8(frame_raw)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# Relevant image region for steering angle prediction
main = frame[s.get_value(Settings.IMAGE_FRONT_BORDER_TOP):s.
get_value(Settings.IMAGE_FRONT_BORDER_BOTTOM),
s.get_value(Settings.IMAGE_FRONT_BORDER_LEFT):s.
get_value(Settings.IMAGE_FRONT_BORDER_RIGHT)]
# Resize the image to the size of the neural network input layer
image = scipy.misc.imresize(main, [66, 200]) / 255.0
# Let the neural network predict the new steering angle
y_eval = model.y.eval(session=self.sess,
feed_dict={
model.x: [image],
model.keep_prob: 1.0
})[0][0]
degrees = y_eval * 180 / scipy.pi
steering = int(round((degrees + 180) / 180 * 32768 /
2)) # Value for vjoy controller
# Set the value of the vjoy joystick to the predicted steering angle
if autopilot:
self.controller_thread.set_angle(steering)
self.statusbar.showMessage("Autopilot active")
else:
self.statusbar.showMessage("Autopilot inactive")
# TODO: Show steering wheel in GUI
M = cv2.getRotationMatrix2D((cols / 2, rows / 2), -degrees, 1)
dst = cv2.warpAffine(img_wheel, M, (cols, rows))
# functions.set_image(dst.copy(), self.steering_wheel)
functions.set_image(main.copy(), self.image_front)
self.controller_thread.set_autopilot(False)
class DatabaseTest(unittest.TestCase):
engine = create_engine("sqlite:///python_models.db", echo=True)
meta = MetaData()
def setUp(self):
self.df = pd.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
columns=['a', 'b', 'c'])
self._data = Data("unittest")
self._data.add_test_table('unittest')
def test_csv_to_df(self):
self.df.to_csv('./tests/unittest.csv')
self._data.csv_to_df()
self.assertEqual(type(pd.DataFrame()), type(self._data.df))
def test_csv_to_db(self):
self.assertTrue(self._data.csv_to_db())
def test_fit_model(self):
with self.assertRaises(RegressionException):
self._data.fit_model(1, Data('ideal'), 'real')
self.df = pd.DataFrame(
[[1.5555555, 2.55555555], [4.5555555, 5.5555555]],
columns=['x', 'y1'])
self.df.to_csv('./tests/unittest.csv')
self._data.csv_to_df()
with self.assertRaises(Exception):
self._data.fit_model(1, None, 'linear')
# test print_table arg
with open('./datasets/ideal.csv', 'r') as idf:
ideal = Data('ideal', _create=False)
ideal.csv_to_df()
self.assertFalse(ideal.is_empty(), 'df obj should be populated')
model = self._data.fit_model(1,
ideal,
'linear',
print_table=True,
table_name='./tests/unittest')
self.assertEqual(type(model),
type(Model(self.df['x'], self.df['y1'], 1)))
self.assertEqual(model.x[0], self.df['x'][0])
def tearDown(self):
try:
self._data.drop_test_table()
except AttributeError:
print('table not dropped')
def setUp(self):
self.df = pd.DataFrame(np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]),
columns=['a', 'b', 'c'])
self._data = Data("unittest")
self._data.add_test_table('unittest')