def run(self):
while not self.lock.wait(timeout=self.interval):
url = self.soldier.superior_ep + "subordinates/" + self.soldier.id
res, err = rest.get(url, etag=self.etag)
if err is not None:
# TODO: エラー処理ちゃんとやる
logger.error('In HeartBeat, failed to post report')
logger.error('> err: {0}', err)
break
if res.status_code == 304:
continue
self.etag = res.headers['ETag']
info = SoldierInfo.make(res.json()['info'])
logger.info([str(m) for m in info.orders])
# 以前に受理されているorderにはあるが新規のorderリストには無い
# （消された）orderを消す
oid_list = [o.get_id() for o in info.orders]
for old_oid in self.soldier.orders.keys():
if old_oid not in oid_list:
self.soldier.remove_order(old_oid)
for m in info.orders:
self.soldier.accept_order(m)
self.soldier.shutdown()
def load_features(tokenizer, data_dir, max_seq_length, evaluate):
processor = DataProcessor()
logger.info("Creating features from dataset file at %s", data_dir)
label_list = processor.get_labels()
examples = processor.get_dev_examples(data_dir) if evaluate else processor.get_train_examples(data_dir)
features = build_features(examples, label_list, max_seq_length, tokenizer)
return features
def run_training(train:str, test:str, model_dir:str, output_data_dir:str, **hyperparams):
logger.info("Running training with training data: '%s', test data: '%s', model dir: '%s', output data: %s",
train, test, model_dir, output_data_dir)
# install_dependencies()
params = model.DEFAULT_HYPERPARAMETERS.copy()
params.update(hyperparams)
logger.info("Hyperparameters: %s", params)
do_lower_case = params.pop("do_lower_case")
model_pipeline, tokenizer = model.build_model(do_lower_case=do_lower_case,
num_labels=(len(DataProcessor().get_labels())))
max_seq_length = params.pop("max_seq_length")
train_dataset = load_and_cache_examples(tokenizer,
train,
max_seq_length,
evaluate=False)
eval_dataset = load_and_cache_examples(tokenizer,
test,
max_seq_length,
evaluate=True)
train_model(model_pipeline,
tokenizer,
train_dataset,
eval_dataset,
output_data_dir,
model_dir,
**params)
results = model.evaluate_checkpoints(eval_dataset, output_data_dir, eval_all_checkpoints=True)
def awake(self, rec_ep: str, heartbeat_rate: int):
from model import LeaderInfo
# 上官を解決する
url = rec_ep + 'department/squad/leader?soldier_id=' + self.id
res, err = rest.get(url)
if err is not None:
return False
self.place = res.json()["place"]
superior = LeaderInfo.make(res.json()['leader'])
self.superior_ep = superior.endpoint
logger.info("superior was resolved: id={0}".format(superior.id))
# 分隊に加入する
url = self.superior_ep + "subordinates"
info = SoldierInfo(id=self.id,
name=self.name,
place=self.place,
weapons=list(self.weapons.keys()),
orders=list(self.orders.values()))
res, err = rest.post(url, json=info.to_dict())
if err is not None:
return False
logger.info("joined to squad: leader_id: {0}".format(superior.id))
# orderを取得する
self.heartbeat_thread.interval = heartbeat_rate
self.heartbeat_thread.start()
return True
def awake(self, rec_ep: str, heartbeat_rate: int):
from model import LeaderInfo
# 上官を解決する
url = rec_ep + 'department/squad/leader?soldier_id=' + self.id
res, err = rest.get(url)
if err is not None:
return False
self.place = res.json()["place"]
superior = LeaderInfo.make(res.json()['leader'])
self.superior_ep = superior.endpoint
logger.info("superior was resolved: id={0}".format(superior.id))
# 分隊に加入する
url = self.superior_ep + "subordinates"
info = SoldierInfo(
id=self.id,
name=self.name,
place=self.place,
weapons=list(self.weapons.keys()),
orders=list(self.orders.values()))
res, err = rest.post(url, json=info.to_dict())
if err is not None:
return False
logger.info("joined to squad: leader_id: {0}".format(superior.id))
# orderを取得する
self.heartbeat_thread.interval = heartbeat_rate
self.heartbeat_thread.start()
return True
def run(self):
while not self.lock.wait(timeout=self.interval):
url = self.soldier.superior_ep + "subordinates/" + self.soldier.id
res, err = rest.get(url, etag=self.etag)
if err is not None:
# TODO: エラー処理ちゃんとやる
logger.error('In HeartBeat, failed to post report')
logger.error('> err: {0}', err)
break
if res.status_code == 304:
continue
self.etag = res.headers['ETag']
info = SoldierInfo.make(res.json()['info'])
logger.info([str(m) for m in info.orders])
# 以前に受理されているorderにはあるが新規のorderリストには無い
# （消された）orderを消す
oid_list = [o.get_id() for o in info.orders]
for old_oid in self.soldier.orders.keys():
if old_oid not in oid_list:
self.soldier.remove_order(old_oid)
for m in info.orders:
self.soldier.accept_order(m)
self.soldier.shutdown()
def __log_buy(self, btc_bought, usdt_cost):
logger.next_line()
self.__log_current_level_market_price()
logger.info(
logger.BUY,
f"Bought {btc_bought} BTC for a total of {usdt_cost} USDT.")
self.__log_status()
def __log_sell(self, btc_sold, usdt_obtained, profit_gain):
logger.next_line()
self.__log_current_level_market_price()
logger.info(
logger.SELL,
f"Sold {btc_sold} BTC for a total of {usdt_obtained} USDT.")
logger.info(logger.PROFIT, f"Profit gained: + {profit_gain} USDT.")
self.__log_status()
def train_model(model_pipeline, tokenizer, train_dataset, eval_dataset, output_dir:str, model_dir:str, **hyperparams:Dict):
logger.info("Start training")
model.train(model_pipeline, tokenizer, train_dataset, eval_dataset, output_dir, **hyperparams)
utils.write_model(model_pipeline, tokenizer, model_dir)
logger.info("Training done")
def awake(self, rec_ep: str):
self.recruiter_ep = rec_ep
url = "{0}commanders/{1}".format(rec_ep, self.id)
res, err = rest.put(url, json=self.generate_info().to_dict())
if err is not None:
return False
new_info = CommanderInfo.make(res.json()["commander"])
self.place = new_info.place
logger.info("register commander to recruiter: success")
return True
def awake(self, rec_ep: str):
self.recruiter_ep = rec_ep
url = "{0}commanders/{1}".format(rec_ep, self.id)
res, err = rest.put(url, json=self.generate_info().to_dict())
if err is not None:
return False
new_info = CommanderInfo.make(res.json()["commander"])
self.place = new_info.place
logger.info("register commander to recruiter: success")
return True
def load_and_cache_examples(tokenizer, data_dir, max_seq_length, evaluate=False):
# Load data features from cache or dataset file
cached_features_file = get_cache_file_path(data_dir, max_seq_length, evaluate)
if os.path.exists(cached_features_file):
features = load_cached_features(cached_features_file)
else:
features = load_features(tokenizer, data_dir, max_seq_length, evaluate)
logger.info("Saving features into cached file %s", cached_features_file)
torch.save(features, cached_features_file)
# Convert to Tensors and build dataset
all_input_ids, all_input_mask, all_segment_ids, all_label_ids = convert_to_tensors(features)
dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)
return dataset
def __on_start(self):
logger.info(logger.START, "Bot started!")
# Initial buy excecuted
self.usdt -= self.INITIAL_BTC_INVERSION
self.btc = self.INITIAL_BTC
self.__log_buy(self.INITIAL_BTC, self.INITIAL_BTC_INVERSION)
# Upper levels
for lvl in range(self.__initial_level() + 1, 2 * self.LEVELS + 1):
to_sell = self.INITIAL_BTC / self.LEVELS
id = self.exchange.set_limit_sell_order(to_sell,
self.__level_price(lvl))
self.active_orders[lvl] = (id, to_sell)
# Lower levels
for lvl in range(0, self.__initial_level()):
id = self.exchange.set_limit_buy_order(self.PER_LEVEL_BUY,
self.__level_price(lvl))
self.active_orders[lvl] = (id, )
def __log_config(self):
profit_range = self.__estimated_profit_per_sell_range()
config_description = """
INVERSION: $ %f USDT
INITIAL BUY: $ %f USDT
RANGE: %f %%
STARTING PRICE: $ %f USDT
UPPER BOUND: $ %f USDT
LOWER BOUND: $ %f USDT
TOTAL LEVELS: %d
LEVEL HEIGHT: $ %f USDT
PER LEVEL BUY: $ %f USDT
ESTIMATED PROFIT PER SELL: + %f%% - %f%% (%f USDT - %f USDT)
""" % (self.INVERSION, self.INITIAL_BTC_INVERSION, self.DELTA,
self.STARTING_PRICE, self.UPPER_BOUND, self.LOWER_BOUND,
self.LEVELS * 2, self.LEVEL_HEIGHT, self.PER_LEVEL_BUY,
profit_range[0], profit_range[1],
profit_range[0] * self.PER_LEVEL_BUY / 100,
profit_range[1] * self.PER_LEVEL_BUY / 100)
logger.info(logger.INIT, config_description)
def accept_report(self, sub_id, report):
if not self.check_subordinate(sub_id):
return False
if report.purpose == "_error":
msg = report.values[0]["msg"]
logger.info(">> error report is received from {0}".format(sub_id))
logger.info(msg)
self.push_error("leader-{0}'s error: {1}".format(sub_id, msg))
return True
if report.purpose in self.campaigns:
campaign = self.campaigns[report.purpose]
if "mongodb://" in campaign.destination:
push = MongoPush(campaign.destination)
push_data = []
for work in report.values:
push_data.extend([{
"purpose":
campaign.purpose,
"place":
"{0}.{1}".format(report.place, work["place"]),
"time":
work["time"],
"data":
v
} for v in work["values"]])
push.push_values(push_data)
logger.info("accept_report: {0}".format(push_data))
self.report_cache.append(report)
return True
def load_config(self, filename):
"""
人事情報のconfigファイルを読み込んでパラメータセットする
:param str filename: yaml形式で書かれた設定ファイルのファイル名
"""
file = open(filename, 'r')
data = yaml.load(file)
file.close()
for tr in data['troops']:
com_id = tr['commander']
self.places[com_id] = tr["place"]
subs = tr['subordinates']
self.TroopList[com_id] = subs
for sq in data['squads']:
lea_id = sq['leader']
self.places[lea_id] = sq["place"]
subs = []
for s in sq["subordinates"]:
subs.append(s["soldier"])
self.places[s["soldier"]] = s["place"]
self.SquadList[lea_id] = subs
logger.info('load_config done')
logger.info('Troops: {0}'.format(self.TroopList))
logger.info('Squads: {0}'.format(self.SquadList))
def accept_report(self, sub_id, report):
if not self.check_subordinate(sub_id):
return False
if report.purpose == "_error":
msg = report.values[0]["msg"]
logger.info(">> error report is received from {0}".format(sub_id))
logger.info(msg)
self.push_error("leader-{0}'s error: {1}".format(sub_id, msg))
return True
if report.purpose in self.campaigns:
campaign = self.campaigns[report.purpose]
if "mongodb://" in campaign.destination:
push = MongoPush(campaign.destination)
push_data = []
for work in report.values:
push_data.extend([{
"purpose": campaign.purpose,
"place": "{0}.{1}".format(report.place, work["place"]),
"time": work["time"],
"data": v
} for v in work["values"]])
push.push_values(push_data)
logger.info("accept_report: {0}".format(push_data))
self.report_cache.append(report)
return True
def accept_campaign(self, campaign: Campaign):
# Campaignの更新であれば（=IDが同じであれば）既存のものを消す
if campaign.get_id() in self.campaigns:
self.remove_campaign(campaign.get_id())
# 部下のMissionを生成・割り当てる
target_subs = []
if campaign.place == "All":
target_subs = list(self.subordinates.keys())
m_base = Mission(author='',
place='All',
purpose=campaign.get_id(),
requirement=campaign.requirement,
trigger=campaign.trigger)
for t_id in target_subs:
mission = copy.deepcopy(m_base)
mission.author = t_id
self.subordinates[t_id].missions.append(mission)
logger.info(">> got campaign:")
logger.info(json.dumps(campaign.to_dict(), sort_keys=True, indent=2))
self.campaigns[campaign.get_id()] = campaign
return campaign
def awake(self, rec_ep: str, heartbeat_rate: int):
from model import CommanderInfo
# 上官を解決する
url = rec_ep + 'department/troop/commander?leader_id=' + self.id
res, err = rest.get(url)
if err is not None:
return False
self.place = res.json()["place"]
superior = CommanderInfo.make(res.json()['commander'])
self.superior_ep = superior.endpoint
logger.info("superior was resolved: id={0}".format(superior.id))
# 部隊に加入する
url = self.superior_ep + "subordinates"
res, err = rest.post(url, json=self.generate_info().to_dict())
if err is not None:
return False
logger.info("joined to squad: commander_id: {0}".format(superior.id))
# missionを取得する
self.start_heartbeat(heartbeat_rate)
return True
def awake(self, rec_ep: str, heartbeat_rate: int):
from model import CommanderInfo
# 上官を解決する
url = rec_ep + 'department/troop/commander?leader_id=' + self.id
res, err = rest.get(url)
if err is not None:
return False
self.place = res.json()["place"]
superior = CommanderInfo.make(res.json()['commander'])
self.superior_ep = superior.endpoint
logger.info("superior was resolved: id={0}".format(superior.id))
# 部隊に加入する
url = self.superior_ep + "subordinates"
res, err = rest.post(url, json=self.generate_info().to_dict())
if err is not None:
return False
logger.info("joined to squad: commander_id: {0}".format(superior.id))
# missionを取得する
self.start_heartbeat(heartbeat_rate)
return True
def accept_campaign(self, campaign: Campaign):
# Campaignの更新であれば（=IDが同じであれば）既存のものを消す
if campaign.get_id() in self.campaigns:
self.remove_campaign(campaign.get_id())
# 部下のMissionを生成・割り当てる
target_subs = []
if campaign.place == "All":
target_subs = list(self.subordinates.keys())
m_base = Mission(author='',
place='All',
purpose=campaign.get_id(),
requirement=campaign.requirement,
trigger=campaign.trigger)
for t_id in target_subs:
mission = copy.deepcopy(m_base)
mission.author = t_id
self.subordinates[t_id].missions.append(mission)
logger.info(">> got campaign:")
logger.info(json.dumps(campaign.to_dict(), sort_keys=True, indent=2))
self.campaigns[campaign.get_id()] = campaign
return campaign
def on_exit(self):
logger.info(logger.EXIT, "Exiting program.")
self.__log_status()
for order_id in self.active_orders.values():
self.exchange.cancel_order(order_id[0])
def __log_current_level_market_price(self):
logger.info(logger.MARKET,
f"{self.__current_level_price()} USDT / BTC")
def __log_status(self):
logger.info(
logger.STATUS,
f"USDT: {self.usdt} BTC: {self.btc} PROFIT: {self.profit} USDT CURRENT BALANCE: {self.__balance()} USDT"
)
def load_cached_features(cached_features_file):
logger.info("Loading features from cached file %s", cached_features_file)
features = torch.load(cached_features_file)
return features
def cancel_order(self, order_id):
logger.info(logger.ORDER_CANCEL, f"Order {order_id} was canceled.")
