def convert_file(in_filepath, out_dir, es_index):
with open('data/doc_type.pkl', 'rb') as data:
dtype = pickle.load(data)
data_parser = {}
for k in dtype:
data_parser[k] = eval(dtype[k])
data_parser['chr'] = str
data_parser['GWAS_catalog_pubmedid'] = str
data_parser['GRASP_PMID'] = str
error = open("error.log", "w")
count = 0
header = ''
da_list = []
with open(in_filepath) as fp:
row = fp.readline().rstrip()
out_filename = 1
header = row.split("\t")
while row:
row = fp.readline().rstrip()
if not row:
continue
cols = row.split("\t")
count += 1
data = {
"_index": es_index,
"_id": id_gen(cols),
"_source": parse_line(cols, data_parser, header)
}
try:
data = {
"_index": es_index,
"_id": id_gen(cols),
"_source": parse_line(cols, data_parser, header)
}
da_list.append(data)
except:
error.write(cols)
if count % 50000 == 0:
write_to_json(ospath.join(out_dir, str(out_filename) + '.json'), da_list)
out_filename += 1
da_list = []
tmp_print("import {}".format(count), "spend time:",
time.time() - start_time, "s")
write_to_json(ospath.join(out_dir, str(out_filename) + '.json'), da_list)
print("import records ", count)
print("used time", time.time() - start_time, "s")
def convert_file(in_filename, work_dir, es_index):
with open('data/doc_type.pkl', 'rb') as data:
dtype = pickle.load(data)
data_parser = {}
for k in dtype:
data_parser[k] = eval(dtype[k])
data_parser['chr'] = str
data_parser['GWAS_catalog_pubmedid'] = str
data_parser['GRASP_PMID'] = str
error = open("error.log", "w")
count = 0
header = ''
da_list = []
out_path = './' + work_dir + '/'+\
in_filename.split('/')[-1].split('.')[0] + '/'
os.makedirs(out_path, exist_ok=True)
f = gzip.open(in_filename, mode='rb')
out_file_name = '1'
for i in f:
i = i.decode()
count += 1
if not header:
header = i.rstrip().split("\t")
continue
data = {
"_index": es_index,
"_id": id_gen(i),
"_source": parse_line(i, data_parser, header)
}
try:
data = {
"_index": es_index,
"_id": id_gen(i),
"_source": parse_line(i, data_parser, header)
}
da_list.append(data)
except:
error.write(i)
if count % 50000 == 0:
write_data(open(out_path + out_file_name + '.json', 'w'), da_list)
out_file_name = str(int(out_file_name) + 1)
da_list = []
tmp_print("import {}".format(count), "spend time:",
time.time() - start_time, "s")
write_data(open(out_path + out_file_name + '.json', 'w'), da_list)
print("")
print("import record ", count)
print("used time", time.time() - start_time, "s")
def tests():
over_id = id_gen('overfit')
base = {
'initial_learning_rate': 0.01,
'decay_factor': 0.99,
'regularization': 0,
'target': 'overfit'
}
# One layer
yield create_and_train([], False, momentum=.8, train_id=next(over_id), **base)
# Two layers
# With or without batch norm
for bn in [False, True]:
# Different momentum values
for m in [.5, .7, .9]:
yield create_and_train([50], batch_normalized=bn, momentum=m,
train_id=next(over_id), **base)
# Three layers
# With or without batch norm
for bn in [False, True]:
yield create_and_train([50, 30], batch_normalized=bn, momentum=.8,
train_id=next(over_id), **base)
# Four layers
# With or without batch norm
for bn in [False, True]:
yield create_and_train([50, 30, 15], batch_normalized=bn, momentum=.8,
train_id=next(over_id), **base)
def __init__(self, expiration=5):
self.count = 0
self.activity = ['walk']
self.expiration = expiration
self.id = utils.id_gen()
self.q = deque(maxlen=10)
return
def __init__(self, expiration=5):
self.count = 0
self.eps = 1e-6
self.activity = ''
self.expiration = expiration
self.id = utils.id_gen()
self.q = deque(maxlen=10)
self.cubit_q = deque(maxlen=50)
return
def convert_file(in_filename):
count = 0
header = ''
da_list = []
out_path = './json/' + out_folder + \
in_filename.split('/')[-1].split('.')[0] + '/'
os.makedirs(out_path, exist_ok=True)
f = gzip.open(in_filename, mode='rb')
out_file_name = '1'
for i in f:
i = i.decode()
count += 1
if not header:
header = i.rstrip().split("\t")
continue
data = {
"_index": "vs-index",
"_id": id_gen(i),
"_source": parse_line(i, data_parser, header)
}
try:
data = {
"_index": "vs-index",
"_id": id_gen(i),
"_source": parse_line(i, data_parser, header)
}
da_list.append(data)
except:
error.write(i)
if count % 50000 == 0:
write_data(open(out_path + out_file_name + '.json', 'w'), da_list)
out_file_name = str(int(out_file_name) + 1)
da_list = []
tmp_print("import {}".format(count), "spend time:",
time.time() - start_time, "s")
write_data(open(out_path + out_file_name + '.json', 'w'), da_list)
print("")
print("import record ", count)
print("used time", time.time() - start_time, "s")
def tests():
fast_id = id_gen('fast')
base = {
'hidden_layer_sizes': [50],
'decay_factor': 0.99,
'regularization': 0.0001,
'momentum': 0.8,
'target': 'fine'
}
for bn in [False, True]:
for ilr in [0.40573, 0.029202, 0.003215]:
yield create_and_train(batch_normalized=bn,
initial_learning_rate=ilr,
**base,
train_id=next(fast_id))
def __init__(self, expiration=5, nbs=5, scale=1.1, inset=150, min_size=10, model_file=face_mdl):
self.count = 0
self.eps = 1e-6
self.activity = []
self.expiration = expiration
self.id = utils.id_gen()
self.q = deque(maxlen=10)
self.cubit_q = deque(maxlen=50)
self.skeleton_color = tuple([random.randint(0, 255) for _ in range(3)]) #skeleton_color
self.faces = []
self.detector = cv2.CascadeClassifier(model_file)
self.nbs = nbs
self.scale = scale
self.inset = inset
self.min_size = (min_size, min_size)
return
def tests():
deep_id = id_gen('deep')
base = {
'hidden_layer_sizes': [50, 30],
'decay_factor': 0.99,
'momentum': 0.8
}
for bn in [False, True]:
yield create_and_train(batch_normalized=bn,
regularization=0,
initial_learning_rate=0.01,
**base,
train_id=next(deep_id),
target='fine')
# Coarse search with batch norm (random values)
for reg in [0.000192, 0.000672, 0.003772, 0.032430, 0.266132]:
for ilr in [0.40573, 0.029202, 0.012721, 0.003215]:
yield create_and_train(batch_normalized=True,
regularization=reg,
initial_learning_rate=ilr,
**base,
train_id=next(deep_id),
target='coarse')
# Fine search with batch norm
for reg in [0.012361, 0.032430, 0.053913]:
for ilr in [0.043773, 0.029202, 0.016231, 0.001051]:
yield create_and_train(batch_normalized=True,
regularization=reg,
initial_learning_rate=ilr,
**base,
train_id=next(deep_id),
target='fine')
# Last one
yield create_and_train(batch_normalized=True,
regularization=0.0001,
initial_learning_rate=0.029202,
**base,
train_id=next(deep_id),
target='final')
folder = ''
out_path = './json/' + folder + in_filename.split('/')[-1].split('.')[0] + '/'
os.makedirs(out_path, exist_ok=True)
f = gzip.open(in_filename, mode='rb')
out_file_name = '1'
for i in f:
i = i.decode()
count += 1
if not header:
header = i.rstrip().split("\t")
continue
data = {
"_index": "vs-index",
"_id": id_gen(i),
"_source": parse_line(i, data_parser, header)
}
try:
data = {
"_index": "vs-index",
"_id": id_gen(i),
"_source": parse_line(i, data_parser, header)
}
da_list.append(data)
except:
error.write(i)
if count % 50000 == 0:
write_data(open(out_path + out_file_name + '.json', 'w'), da_list)
out_file_name = str(int(out_file_name) + 1)
da_list = []
class Order(object):
order_id_gen = id_gen(int(time.time()))
__repr__ = property_repr
def __init__(self):
self._order_id = None
self._calendar_dt = None
self._trading_dt = None
self._quantity = None
self._order_book_id = None
self._side = None
self._position_effect = None
self._message = None
self._filled_quantity = None
self._status = None
self._frozen_price = None
self._type = None
self._avg_price = None
self._transaction_cost = None
@staticmethod
def _enum_to_str(v):
return v.name
@staticmethod
def _str_to_enum(enum_class, s):
return enum_class.__members__[s]
def get_state(self):
return {
'order_id':
self._order_id,
'calendar_dt':
self._calendar_dt,
'trading_dt':
self._trading_dt,
'order_book_id':
self._order_book_id,
'quantity':
self._quantity,
'side':
self._enum_to_str(self._side),
'position_effect':
self._enum_to_str(self._position_effect)
if self._position_effect is not None else None,
'message':
self._message,
'filled_quantity':
self._filled_quantity,
'status':
self._enum_to_str(self._status),
'frozen_price':
self._frozen_price,
'type':
self._enum_to_str(self._type),
}
def set_state(self, d):
self._order_id = d['order_id']
self._calendar_dt = d['calendar_dt']
self._trading_dt = d['trading_dt']
self._order_book_id = d['order_book_id']
self._quantity = d['quantity']
self._side = self._str_to_enum(SIDE, d['side'])
if d['position_effect'] is None:
self._position_effect = None
else:
self._position_effect = self._str_to_enum(POSITION_EFFECT,
d['position_effect'])
self._message = d['messages']
self._filled_quantity = d['filled_quantity']
self._status = self._str_to_enum(ORDER_STATUS, d['status'])
self._frozen_price = d['frozen_price']
self._type = self._str_to_enum(ORDER_TYPE, d['type'])
@classmethod
def __from_create__(cls, calendar_dt, trading_dt, order_book_id, quantity,
side, style, position_effect):
order = cls()
order._order_id = next(order.order_id_gen)
order._calendar_dt = calendar_dt
order._trading_dt = trading_dt
order._quantity = quantity
order._order_book_id = order_book_id
order._side = side
order._position_effect = position_effect
order._message = ""
order._filled_quantity = 0
order._status = ORDER_STATUS.PENDING_NEW
if isinstance(style, LimitOrder):
order._frozen_price = style.get_limit_price()
order._type = ORDER_TYPE.LIMIT
else:
order._frozen_price = 0.
order._type = ORDER_TYPE.MARKET
order._avg_price = 0
order._transaction_cost = 0
return order
@property
def order_id(self):
"""
[int] 唯一标识订单的id
"""
return self._order_id
@property
def trading_datetime(self):
"""
[datetime.datetime] 订单的交易日期（对应期货夜盘）
"""
return self._trading_dt
@property
def datetime(self):
"""
[datetime.datetime] 订单创建时间
"""
return self._calendar_dt
@property
def quantity(self):
"""
[int] 订单数量
"""
return self._quantity
@property
def unfilled_quantity(self):
"""
[int] 订单未成交数量
"""
return self._quantity - self._filled_quantity
@property
def order_book_id(self):
"""
[str] 合约代码
"""
return self._order_book_id
@property
def side(self):
"""
[SIDE] 订单方向
"""
return self._side
@property
def position_effect(self):
"""
[POSITION_EFFECT] 订单开平（期货专用）
"""
return self._position_effect
@property
def message(self):
"""
[str] 信息。比如拒单时候此处会提示拒单原因
"""
return self._message
@property
def filled_quantity(self):
"""
[int] 订单已成交数量
"""
return self._filled_quantity
@property
def status(self):
"""
[ORDER_STATUS] 订单状态
"""
return self._status
@property
def price(self):
"""
[float] 订单价格，只有在订单类型为'限价单'的时候才有意义
"""
return 0 if self.type == ORDER_TYPE.MARKET else self._frozen_price
@property
def type(self):
"""
[ORDER_TYPE] 订单类型
"""
return self._type
@property
def avg_price(self):
"""
[float] 成交均价
"""
return self._avg_price
@property
def transaction_cost(self):
"""
[float] 费用
"""
return self._transaction_cost
@property
def frozen_price(self):
"""
[float] 冻结价格
"""
return self._frozen_price
def is_final(self):
return self._status not in {
ORDER_STATUS.PENDING_NEW, ORDER_STATUS.ACTIVE,
ORDER_STATUS.PENDING_CANCEL
}
def is_active(self):
return self.status == ORDER_STATUS.ACTIVE
def active(self):
self._status = ORDER_STATUS.ACTIVE
def set_pending_cancel(self):
if not self.is_final():
self._status = ORDER_STATUS.PENDING_CANCEL
def fill(self, trade):
quantity = trade.last_quantity
assert self.filled_quantity + quantity <= self.quantity
new_quantity = self._filled_quantity + quantity
self._avg_price = (self._avg_price * self._filled_quantity +
trade.last_price * quantity) / new_quantity
self._transaction_cost += trade.commission + trade.tax
self._filled_quantity = new_quantity
if self.unfilled_quantity == 0:
self._status = ORDER_STATUS.FILLED
def mark_rejected(self, reject_reason):
if not self.is_final():
self._message = reject_reason
self._status = ORDER_STATUS.REJECTED
user_system_log.warn(reject_reason)
def mark_cancelled(self, cancelled_reason, user_warn=True):
if not self.is_final():
self._message = cancelled_reason
self._status = ORDER_STATUS.CANCELLED
if user_warn:
user_system_log.warn(cancelled_reason)
def set_frozen_price(self, value):
self._frozen_price = value
def __simple_object__(self):
return properties(self)
output_dir = 'out'
models_dir = 'models'
tensorboard_dir = 'tensorboard'
mkdir(tensorboard_dir)
BATCH_SIZE = 16
THRESHOLD = 75.0 # Save only models with accuracy above the threshold
# Uncomment to force CPU run
# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152
# os.environ["CUDA_VISIBLE_DEVICES"] = ""
# DEVELOPMENT ENV: set seed to have reproducible result
from tensorflow.random import set_seed
set_seed(3)
ID = id_gen() #Why not datetime?
tf_log_name = "{}_{}".format(ID, log_info)
tb_base_dir = os.path.join(tensorboard_dir, tf_log_name)
mkdir(tb_base_dir)
print("[Info] Tensorflow is using GPU: {}".format(tf.test.is_gpu_available()))
# LOADING DATASET
# print("[Info] Loading Dataset...")
# IF FIRST TIME, RUN THESE LINES:
# x_train, y_train = io_manager.load_dataset_from_images(dataset_dir)
# io_manager.save_dataset_npz(x_train, y_train, 'preprocessed_1')
# IF ALREADY SAVED NPZ, RUN THIS LINE
if PREPROCESSING == 1:
print("[Info] Loading Dataset - Simple Color Preprocessing")
class Trade(object):
__repr__ = property_repr
trade_id_gen = id_gen(int(time.time()))
def __init__(self):
self._calendar_dt = None
self._trading_dt = None
self._price = None
self._amount = None
self._order_id = None
self._commission = None
self._tax = None
self._trade_id = None
self._close_today_amount = None
self._side = None
self._position_effect = None
self._order_book_id = None
self._frozen_price = None
@classmethod
def __from_create__(cls,
order_id,
calendar_dt,
trading_dt,
price,
amount,
side,
position_effect,
order_book_id,
commission=0.,
tax=0.,
trade_id=None,
close_today_amount=0,
frozen_price=0):
trade = cls()
trade._calendar_dt = calendar_dt
trade._trading_dt = trading_dt
trade._price = price
trade._amount = amount
trade._order_id = order_id
trade._commission = commission
trade._tax = tax
trade._trade_id = trade_id if trade_id is not None else next(
trade.trade_id_gen)
trade._close_today_amount = close_today_amount
trade._side = side
trade._position_effect = position_effect
trade._order_book_id = order_book_id
trade._frozen_price = frozen_price
return trade
@property
def order_book_id(self):
return self._order_book_id
@property
def trading_datetime(self):
return self._trading_dt
@property
def datetime(self):
return self._calendar_dt
@property
def order_id(self):
return self._order_id
@property
def last_price(self):
return self._price
@property
def last_quantity(self):
return self._amount
@property
def commission(self):
return self._commission
@property
def tax(self):
return self._tax
@property
def transaction_cost(self):
return self._tax + self._commission
@property
def side(self):
return self._side
@property
def position_effect(self):
return self._position_effect
@property
def exec_id(self):
return self._trade_id
@property
def frozen_price(self):
return self._frozen_price
@property
def close_today_amount(self):
return self._close_today_amount
def __simple_object__(self):
return properties(self)
