def convert_file(in_filepath, out_dir, es_index): with open('data/doc_type.pkl', 'rb') as data: dtype = pickle.load(data) data_parser = {} for k in dtype: data_parser[k] = eval(dtype[k]) data_parser['chr'] = str data_parser['GWAS_catalog_pubmedid'] = str data_parser['GRASP_PMID'] = str error = open("error.log", "w") count = 0 header = '' da_list = [] with open(in_filepath) as fp: row = fp.readline().rstrip() out_filename = 1 header = row.split("\t") while row: row = fp.readline().rstrip() if not row: continue cols = row.split("\t") count += 1 data = { "_index": es_index, "_id": id_gen(cols), "_source": parse_line(cols, data_parser, header) } try: data = { "_index": es_index, "_id": id_gen(cols), "_source": parse_line(cols, data_parser, header) } da_list.append(data) except: error.write(cols) if count % 50000 == 0: write_to_json(ospath.join(out_dir, str(out_filename) + '.json'), da_list) out_filename += 1 da_list = [] tmp_print("import {}".format(count), "spend time:", time.time() - start_time, "s") write_to_json(ospath.join(out_dir, str(out_filename) + '.json'), da_list) print("import records ", count) print("used time", time.time() - start_time, "s")
def convert_file(in_filename, work_dir, es_index): with open('data/doc_type.pkl', 'rb') as data: dtype = pickle.load(data) data_parser = {} for k in dtype: data_parser[k] = eval(dtype[k]) data_parser['chr'] = str data_parser['GWAS_catalog_pubmedid'] = str data_parser['GRASP_PMID'] = str error = open("error.log", "w") count = 0 header = '' da_list = [] out_path = './' + work_dir + '/'+\ in_filename.split('/')[-1].split('.')[0] + '/' os.makedirs(out_path, exist_ok=True) f = gzip.open(in_filename, mode='rb') out_file_name = '1' for i in f: i = i.decode() count += 1 if not header: header = i.rstrip().split("\t") continue data = { "_index": es_index, "_id": id_gen(i), "_source": parse_line(i, data_parser, header) } try: data = { "_index": es_index, "_id": id_gen(i), "_source": parse_line(i, data_parser, header) } da_list.append(data) except: error.write(i) if count % 50000 == 0: write_data(open(out_path + out_file_name + '.json', 'w'), da_list) out_file_name = str(int(out_file_name) + 1) da_list = [] tmp_print("import {}".format(count), "spend time:", time.time() - start_time, "s") write_data(open(out_path + out_file_name + '.json', 'w'), da_list) print("") print("import record ", count) print("used time", time.time() - start_time, "s")
def tests(): over_id = id_gen('overfit') base = { 'initial_learning_rate': 0.01, 'decay_factor': 0.99, 'regularization': 0, 'target': 'overfit' } # One layer yield create_and_train([], False, momentum=.8, train_id=next(over_id), **base) # Two layers # With or without batch norm for bn in [False, True]: # Different momentum values for m in [.5, .7, .9]: yield create_and_train([50], batch_normalized=bn, momentum=m, train_id=next(over_id), **base) # Three layers # With or without batch norm for bn in [False, True]: yield create_and_train([50, 30], batch_normalized=bn, momentum=.8, train_id=next(over_id), **base) # Four layers # With or without batch norm for bn in [False, True]: yield create_and_train([50, 30, 15], batch_normalized=bn, momentum=.8, train_id=next(over_id), **base)
def __init__(self, expiration=5): self.count = 0 self.activity = ['walk'] self.expiration = expiration self.id = utils.id_gen() self.q = deque(maxlen=10) return
def __init__(self, expiration=5): self.count = 0 self.eps = 1e-6 self.activity = '' self.expiration = expiration self.id = utils.id_gen() self.q = deque(maxlen=10) self.cubit_q = deque(maxlen=50) return
def convert_file(in_filename): count = 0 header = '' da_list = [] out_path = './json/' + out_folder + \ in_filename.split('/')[-1].split('.')[0] + '/' os.makedirs(out_path, exist_ok=True) f = gzip.open(in_filename, mode='rb') out_file_name = '1' for i in f: i = i.decode() count += 1 if not header: header = i.rstrip().split("\t") continue data = { "_index": "vs-index", "_id": id_gen(i), "_source": parse_line(i, data_parser, header) } try: data = { "_index": "vs-index", "_id": id_gen(i), "_source": parse_line(i, data_parser, header) } da_list.append(data) except: error.write(i) if count % 50000 == 0: write_data(open(out_path + out_file_name + '.json', 'w'), da_list) out_file_name = str(int(out_file_name) + 1) da_list = [] tmp_print("import {}".format(count), "spend time:", time.time() - start_time, "s") write_data(open(out_path + out_file_name + '.json', 'w'), da_list) print("") print("import record ", count) print("used time", time.time() - start_time, "s")
def tests(): fast_id = id_gen('fast') base = { 'hidden_layer_sizes': [50], 'decay_factor': 0.99, 'regularization': 0.0001, 'momentum': 0.8, 'target': 'fine' } for bn in [False, True]: for ilr in [0.40573, 0.029202, 0.003215]: yield create_and_train(batch_normalized=bn, initial_learning_rate=ilr, **base, train_id=next(fast_id))
def __init__(self, expiration=5, nbs=5, scale=1.1, inset=150, min_size=10, model_file=face_mdl): self.count = 0 self.eps = 1e-6 self.activity = [] self.expiration = expiration self.id = utils.id_gen() self.q = deque(maxlen=10) self.cubit_q = deque(maxlen=50) self.skeleton_color = tuple([random.randint(0, 255) for _ in range(3)]) #skeleton_color self.faces = [] self.detector = cv2.CascadeClassifier(model_file) self.nbs = nbs self.scale = scale self.inset = inset self.min_size = (min_size, min_size) return
def tests(): deep_id = id_gen('deep') base = { 'hidden_layer_sizes': [50, 30], 'decay_factor': 0.99, 'momentum': 0.8 } for bn in [False, True]: yield create_and_train(batch_normalized=bn, regularization=0, initial_learning_rate=0.01, **base, train_id=next(deep_id), target='fine') # Coarse search with batch norm (random values) for reg in [0.000192, 0.000672, 0.003772, 0.032430, 0.266132]: for ilr in [0.40573, 0.029202, 0.012721, 0.003215]: yield create_and_train(batch_normalized=True, regularization=reg, initial_learning_rate=ilr, **base, train_id=next(deep_id), target='coarse') # Fine search with batch norm for reg in [0.012361, 0.032430, 0.053913]: for ilr in [0.043773, 0.029202, 0.016231, 0.001051]: yield create_and_train(batch_normalized=True, regularization=reg, initial_learning_rate=ilr, **base, train_id=next(deep_id), target='fine') # Last one yield create_and_train(batch_normalized=True, regularization=0.0001, initial_learning_rate=0.029202, **base, train_id=next(deep_id), target='final')
folder = '' out_path = './json/' + folder + in_filename.split('/')[-1].split('.')[0] + '/' os.makedirs(out_path, exist_ok=True) f = gzip.open(in_filename, mode='rb') out_file_name = '1' for i in f: i = i.decode() count += 1 if not header: header = i.rstrip().split("\t") continue data = { "_index": "vs-index", "_id": id_gen(i), "_source": parse_line(i, data_parser, header) } try: data = { "_index": "vs-index", "_id": id_gen(i), "_source": parse_line(i, data_parser, header) } da_list.append(data) except: error.write(i) if count % 50000 == 0: write_data(open(out_path + out_file_name + '.json', 'w'), da_list) out_file_name = str(int(out_file_name) + 1) da_list = []
class Order(object): order_id_gen = id_gen(int(time.time())) __repr__ = property_repr def __init__(self): self._order_id = None self._calendar_dt = None self._trading_dt = None self._quantity = None self._order_book_id = None self._side = None self._position_effect = None self._message = None self._filled_quantity = None self._status = None self._frozen_price = None self._type = None self._avg_price = None self._transaction_cost = None @staticmethod def _enum_to_str(v): return v.name @staticmethod def _str_to_enum(enum_class, s): return enum_class.__members__[s] def get_state(self): return { 'order_id': self._order_id, 'calendar_dt': self._calendar_dt, 'trading_dt': self._trading_dt, 'order_book_id': self._order_book_id, 'quantity': self._quantity, 'side': self._enum_to_str(self._side), 'position_effect': self._enum_to_str(self._position_effect) if self._position_effect is not None else None, 'message': self._message, 'filled_quantity': self._filled_quantity, 'status': self._enum_to_str(self._status), 'frozen_price': self._frozen_price, 'type': self._enum_to_str(self._type), } def set_state(self, d): self._order_id = d['order_id'] self._calendar_dt = d['calendar_dt'] self._trading_dt = d['trading_dt'] self._order_book_id = d['order_book_id'] self._quantity = d['quantity'] self._side = self._str_to_enum(SIDE, d['side']) if d['position_effect'] is None: self._position_effect = None else: self._position_effect = self._str_to_enum(POSITION_EFFECT, d['position_effect']) self._message = d['messages'] self._filled_quantity = d['filled_quantity'] self._status = self._str_to_enum(ORDER_STATUS, d['status']) self._frozen_price = d['frozen_price'] self._type = self._str_to_enum(ORDER_TYPE, d['type']) @classmethod def __from_create__(cls, calendar_dt, trading_dt, order_book_id, quantity, side, style, position_effect): order = cls() order._order_id = next(order.order_id_gen) order._calendar_dt = calendar_dt order._trading_dt = trading_dt order._quantity = quantity order._order_book_id = order_book_id order._side = side order._position_effect = position_effect order._message = "" order._filled_quantity = 0 order._status = ORDER_STATUS.PENDING_NEW if isinstance(style, LimitOrder): order._frozen_price = style.get_limit_price() order._type = ORDER_TYPE.LIMIT else: order._frozen_price = 0. order._type = ORDER_TYPE.MARKET order._avg_price = 0 order._transaction_cost = 0 return order @property def order_id(self): """ [int] 唯一标识订单的id """ return self._order_id @property def trading_datetime(self): """ [datetime.datetime] 订单的交易日期(对应期货夜盘) """ return self._trading_dt @property def datetime(self): """ [datetime.datetime] 订单创建时间 """ return self._calendar_dt @property def quantity(self): """ [int] 订单数量 """ return self._quantity @property def unfilled_quantity(self): """ [int] 订单未成交数量 """ return self._quantity - self._filled_quantity @property def order_book_id(self): """ [str] 合约代码 """ return self._order_book_id @property def side(self): """ [SIDE] 订单方向 """ return self._side @property def position_effect(self): """ [POSITION_EFFECT] 订单开平(期货专用) """ return self._position_effect @property def message(self): """ [str] 信息。比如拒单时候此处会提示拒单原因 """ return self._message @property def filled_quantity(self): """ [int] 订单已成交数量 """ return self._filled_quantity @property def status(self): """ [ORDER_STATUS] 订单状态 """ return self._status @property def price(self): """ [float] 订单价格,只有在订单类型为'限价单'的时候才有意义 """ return 0 if self.type == ORDER_TYPE.MARKET else self._frozen_price @property def type(self): """ [ORDER_TYPE] 订单类型 """ return self._type @property def avg_price(self): """ [float] 成交均价 """ return self._avg_price @property def transaction_cost(self): """ [float] 费用 """ return self._transaction_cost @property def frozen_price(self): """ [float] 冻结价格 """ return self._frozen_price def is_final(self): return self._status not in { ORDER_STATUS.PENDING_NEW, ORDER_STATUS.ACTIVE, ORDER_STATUS.PENDING_CANCEL } def is_active(self): return self.status == ORDER_STATUS.ACTIVE def active(self): self._status = ORDER_STATUS.ACTIVE def set_pending_cancel(self): if not self.is_final(): self._status = ORDER_STATUS.PENDING_CANCEL def fill(self, trade): quantity = trade.last_quantity assert self.filled_quantity + quantity <= self.quantity new_quantity = self._filled_quantity + quantity self._avg_price = (self._avg_price * self._filled_quantity + trade.last_price * quantity) / new_quantity self._transaction_cost += trade.commission + trade.tax self._filled_quantity = new_quantity if self.unfilled_quantity == 0: self._status = ORDER_STATUS.FILLED def mark_rejected(self, reject_reason): if not self.is_final(): self._message = reject_reason self._status = ORDER_STATUS.REJECTED user_system_log.warn(reject_reason) def mark_cancelled(self, cancelled_reason, user_warn=True): if not self.is_final(): self._message = cancelled_reason self._status = ORDER_STATUS.CANCELLED if user_warn: user_system_log.warn(cancelled_reason) def set_frozen_price(self, value): self._frozen_price = value def __simple_object__(self): return properties(self)
output_dir = 'out' models_dir = 'models' tensorboard_dir = 'tensorboard' mkdir(tensorboard_dir) BATCH_SIZE = 16 THRESHOLD = 75.0 # Save only models with accuracy above the threshold # Uncomment to force CPU run # os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # see issue #152 # os.environ["CUDA_VISIBLE_DEVICES"] = "" # DEVELOPMENT ENV: set seed to have reproducible result from tensorflow.random import set_seed set_seed(3) ID = id_gen() #Why not datetime? tf_log_name = "{}_{}".format(ID, log_info) tb_base_dir = os.path.join(tensorboard_dir, tf_log_name) mkdir(tb_base_dir) print("[Info] Tensorflow is using GPU: {}".format(tf.test.is_gpu_available())) # LOADING DATASET # print("[Info] Loading Dataset...") # IF FIRST TIME, RUN THESE LINES: # x_train, y_train = io_manager.load_dataset_from_images(dataset_dir) # io_manager.save_dataset_npz(x_train, y_train, 'preprocessed_1') # IF ALREADY SAVED NPZ, RUN THIS LINE if PREPROCESSING == 1: print("[Info] Loading Dataset - Simple Color Preprocessing")
class Trade(object): __repr__ = property_repr trade_id_gen = id_gen(int(time.time())) def __init__(self): self._calendar_dt = None self._trading_dt = None self._price = None self._amount = None self._order_id = None self._commission = None self._tax = None self._trade_id = None self._close_today_amount = None self._side = None self._position_effect = None self._order_book_id = None self._frozen_price = None @classmethod def __from_create__(cls, order_id, calendar_dt, trading_dt, price, amount, side, position_effect, order_book_id, commission=0., tax=0., trade_id=None, close_today_amount=0, frozen_price=0): trade = cls() trade._calendar_dt = calendar_dt trade._trading_dt = trading_dt trade._price = price trade._amount = amount trade._order_id = order_id trade._commission = commission trade._tax = tax trade._trade_id = trade_id if trade_id is not None else next( trade.trade_id_gen) trade._close_today_amount = close_today_amount trade._side = side trade._position_effect = position_effect trade._order_book_id = order_book_id trade._frozen_price = frozen_price return trade @property def order_book_id(self): return self._order_book_id @property def trading_datetime(self): return self._trading_dt @property def datetime(self): return self._calendar_dt @property def order_id(self): return self._order_id @property def last_price(self): return self._price @property def last_quantity(self): return self._amount @property def commission(self): return self._commission @property def tax(self): return self._tax @property def transaction_cost(self): return self._tax + self._commission @property def side(self): return self._side @property def position_effect(self): return self._position_effect @property def exec_id(self): return self._trade_id @property def frozen_price(self): return self._frozen_price @property def close_today_amount(self): return self._close_today_amount def __simple_object__(self): return properties(self)