def run(self): while not self.lock.wait(timeout=self.interval): url = self.soldier.superior_ep + "subordinates/" + self.soldier.id res, err = rest.get(url, etag=self.etag) if err is not None: # TODO: エラー処理ちゃんとやる logger.error('In HeartBeat, failed to post report') logger.error('> err: {0}', err) break if res.status_code == 304: continue self.etag = res.headers['ETag'] info = SoldierInfo.make(res.json()['info']) logger.info([str(m) for m in info.orders]) # 以前に受理されているorderにはあるが新規のorderリストには無い # (消された)orderを消す oid_list = [o.get_id() for o in info.orders] for old_oid in self.soldier.orders.keys(): if old_oid not in oid_list: self.soldier.remove_order(old_oid) for m in info.orders: self.soldier.accept_order(m) self.soldier.shutdown()
def load_features(tokenizer, data_dir, max_seq_length, evaluate): processor = DataProcessor() logger.info("Creating features from dataset file at %s", data_dir) label_list = processor.get_labels() examples = processor.get_dev_examples(data_dir) if evaluate else processor.get_train_examples(data_dir) features = build_features(examples, label_list, max_seq_length, tokenizer) return features
def run_training(train:str, test:str, model_dir:str, output_data_dir:str, **hyperparams): logger.info("Running training with training data: '%s', test data: '%s', model dir: '%s', output data: %s", train, test, model_dir, output_data_dir) # install_dependencies() params = model.DEFAULT_HYPERPARAMETERS.copy() params.update(hyperparams) logger.info("Hyperparameters: %s", params) do_lower_case = params.pop("do_lower_case") model_pipeline, tokenizer = model.build_model(do_lower_case=do_lower_case, num_labels=(len(DataProcessor().get_labels()))) max_seq_length = params.pop("max_seq_length") train_dataset = load_and_cache_examples(tokenizer, train, max_seq_length, evaluate=False) eval_dataset = load_and_cache_examples(tokenizer, test, max_seq_length, evaluate=True) train_model(model_pipeline, tokenizer, train_dataset, eval_dataset, output_data_dir, model_dir, **params) results = model.evaluate_checkpoints(eval_dataset, output_data_dir, eval_all_checkpoints=True)
def awake(self, rec_ep: str, heartbeat_rate: int): from model import LeaderInfo # 上官を解決する url = rec_ep + 'department/squad/leader?soldier_id=' + self.id res, err = rest.get(url) if err is not None: return False self.place = res.json()["place"] superior = LeaderInfo.make(res.json()['leader']) self.superior_ep = superior.endpoint logger.info("superior was resolved: id={0}".format(superior.id)) # 分隊に加入する url = self.superior_ep + "subordinates" info = SoldierInfo(id=self.id, name=self.name, place=self.place, weapons=list(self.weapons.keys()), orders=list(self.orders.values())) res, err = rest.post(url, json=info.to_dict()) if err is not None: return False logger.info("joined to squad: leader_id: {0}".format(superior.id)) # orderを取得する self.heartbeat_thread.interval = heartbeat_rate self.heartbeat_thread.start() return True
def awake(self, rec_ep: str, heartbeat_rate: int): from model import LeaderInfo # 上官を解決する url = rec_ep + 'department/squad/leader?soldier_id=' + self.id res, err = rest.get(url) if err is not None: return False self.place = res.json()["place"] superior = LeaderInfo.make(res.json()['leader']) self.superior_ep = superior.endpoint logger.info("superior was resolved: id={0}".format(superior.id)) # 分隊に加入する url = self.superior_ep + "subordinates" info = SoldierInfo( id=self.id, name=self.name, place=self.place, weapons=list(self.weapons.keys()), orders=list(self.orders.values())) res, err = rest.post(url, json=info.to_dict()) if err is not None: return False logger.info("joined to squad: leader_id: {0}".format(superior.id)) # orderを取得する self.heartbeat_thread.interval = heartbeat_rate self.heartbeat_thread.start() return True
def __log_buy(self, btc_bought, usdt_cost): logger.next_line() self.__log_current_level_market_price() logger.info( logger.BUY, f"Bought {btc_bought} BTC for a total of {usdt_cost} USDT.") self.__log_status()
def __log_sell(self, btc_sold, usdt_obtained, profit_gain): logger.next_line() self.__log_current_level_market_price() logger.info( logger.SELL, f"Sold {btc_sold} BTC for a total of {usdt_obtained} USDT.") logger.info(logger.PROFIT, f"Profit gained: + {profit_gain} USDT.") self.__log_status()
def train_model(model_pipeline, tokenizer, train_dataset, eval_dataset, output_dir:str, model_dir:str, **hyperparams:Dict): logger.info("Start training") model.train(model_pipeline, tokenizer, train_dataset, eval_dataset, output_dir, **hyperparams) utils.write_model(model_pipeline, tokenizer, model_dir) logger.info("Training done")
def awake(self, rec_ep: str): self.recruiter_ep = rec_ep url = "{0}commanders/{1}".format(rec_ep, self.id) res, err = rest.put(url, json=self.generate_info().to_dict()) if err is not None: return False new_info = CommanderInfo.make(res.json()["commander"]) self.place = new_info.place logger.info("register commander to recruiter: success") return True
def load_and_cache_examples(tokenizer, data_dir, max_seq_length, evaluate=False): # Load data features from cache or dataset file cached_features_file = get_cache_file_path(data_dir, max_seq_length, evaluate) if os.path.exists(cached_features_file): features = load_cached_features(cached_features_file) else: features = load_features(tokenizer, data_dir, max_seq_length, evaluate) logger.info("Saving features into cached file %s", cached_features_file) torch.save(features, cached_features_file) # Convert to Tensors and build dataset all_input_ids, all_input_mask, all_segment_ids, all_label_ids = convert_to_tensors(features) dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids) return dataset
def __on_start(self): logger.info(logger.START, "Bot started!") # Initial buy excecuted self.usdt -= self.INITIAL_BTC_INVERSION self.btc = self.INITIAL_BTC self.__log_buy(self.INITIAL_BTC, self.INITIAL_BTC_INVERSION) # Upper levels for lvl in range(self.__initial_level() + 1, 2 * self.LEVELS + 1): to_sell = self.INITIAL_BTC / self.LEVELS id = self.exchange.set_limit_sell_order(to_sell, self.__level_price(lvl)) self.active_orders[lvl] = (id, to_sell) # Lower levels for lvl in range(0, self.__initial_level()): id = self.exchange.set_limit_buy_order(self.PER_LEVEL_BUY, self.__level_price(lvl)) self.active_orders[lvl] = (id, )
def __log_config(self): profit_range = self.__estimated_profit_per_sell_range() config_description = """ INVERSION: $ %f USDT INITIAL BUY: $ %f USDT RANGE: %f %% STARTING PRICE: $ %f USDT UPPER BOUND: $ %f USDT LOWER BOUND: $ %f USDT TOTAL LEVELS: %d LEVEL HEIGHT: $ %f USDT PER LEVEL BUY: $ %f USDT ESTIMATED PROFIT PER SELL: + %f%% - %f%% (%f USDT - %f USDT) """ % (self.INVERSION, self.INITIAL_BTC_INVERSION, self.DELTA, self.STARTING_PRICE, self.UPPER_BOUND, self.LOWER_BOUND, self.LEVELS * 2, self.LEVEL_HEIGHT, self.PER_LEVEL_BUY, profit_range[0], profit_range[1], profit_range[0] * self.PER_LEVEL_BUY / 100, profit_range[1] * self.PER_LEVEL_BUY / 100) logger.info(logger.INIT, config_description)
def accept_report(self, sub_id, report): if not self.check_subordinate(sub_id): return False if report.purpose == "_error": msg = report.values[0]["msg"] logger.info(">> error report is received from {0}".format(sub_id)) logger.info(msg) self.push_error("leader-{0}'s error: {1}".format(sub_id, msg)) return True if report.purpose in self.campaigns: campaign = self.campaigns[report.purpose] if "mongodb://" in campaign.destination: push = MongoPush(campaign.destination) push_data = [] for work in report.values: push_data.extend([{ "purpose": campaign.purpose, "place": "{0}.{1}".format(report.place, work["place"]), "time": work["time"], "data": v } for v in work["values"]]) push.push_values(push_data) logger.info("accept_report: {0}".format(push_data)) self.report_cache.append(report) return True
def load_config(self, filename): """ 人事情報のconfigファイルを読み込んでパラメータセットする :param str filename: yaml形式で書かれた設定ファイルのファイル名 """ file = open(filename, 'r') data = yaml.load(file) file.close() for tr in data['troops']: com_id = tr['commander'] self.places[com_id] = tr["place"] subs = tr['subordinates'] self.TroopList[com_id] = subs for sq in data['squads']: lea_id = sq['leader'] self.places[lea_id] = sq["place"] subs = [] for s in sq["subordinates"]: subs.append(s["soldier"]) self.places[s["soldier"]] = s["place"] self.SquadList[lea_id] = subs logger.info('load_config done') logger.info('Troops: {0}'.format(self.TroopList)) logger.info('Squads: {0}'.format(self.SquadList))
def accept_campaign(self, campaign: Campaign): # Campaignの更新であれば(=IDが同じであれば)既存のものを消す if campaign.get_id() in self.campaigns: self.remove_campaign(campaign.get_id()) # 部下のMissionを生成・割り当てる target_subs = [] if campaign.place == "All": target_subs = list(self.subordinates.keys()) m_base = Mission(author='', place='All', purpose=campaign.get_id(), requirement=campaign.requirement, trigger=campaign.trigger) for t_id in target_subs: mission = copy.deepcopy(m_base) mission.author = t_id self.subordinates[t_id].missions.append(mission) logger.info(">> got campaign:") logger.info(json.dumps(campaign.to_dict(), sort_keys=True, indent=2)) self.campaigns[campaign.get_id()] = campaign return campaign
def awake(self, rec_ep: str, heartbeat_rate: int): from model import CommanderInfo # 上官を解決する url = rec_ep + 'department/troop/commander?leader_id=' + self.id res, err = rest.get(url) if err is not None: return False self.place = res.json()["place"] superior = CommanderInfo.make(res.json()['commander']) self.superior_ep = superior.endpoint logger.info("superior was resolved: id={0}".format(superior.id)) # 部隊に加入する url = self.superior_ep + "subordinates" res, err = rest.post(url, json=self.generate_info().to_dict()) if err is not None: return False logger.info("joined to squad: commander_id: {0}".format(superior.id)) # missionを取得する self.start_heartbeat(heartbeat_rate) return True
def on_exit(self): logger.info(logger.EXIT, "Exiting program.") self.__log_status() for order_id in self.active_orders.values(): self.exchange.cancel_order(order_id[0])
def __log_current_level_market_price(self): logger.info(logger.MARKET, f"{self.__current_level_price()} USDT / BTC")
def __log_status(self): logger.info( logger.STATUS, f"USDT: {self.usdt} BTC: {self.btc} PROFIT: {self.profit} USDT CURRENT BALANCE: {self.__balance()} USDT" )
def load_cached_features(cached_features_file): logger.info("Loading features from cached file %s", cached_features_file) features = torch.load(cached_features_file) return features
def cancel_order(self, order_id): logger.info(logger.ORDER_CANCEL, f"Order {order_id} was canceled.")