def start(self):
self.trading_engine.start()
counter = 0
while True:
start_time = time.time()
logger.info(
"-------count: {}, algos:{}--------".format(counter, self.algos.keys()))
for algo in self.algos:
self.algos[algo].prerun(t=datetime.now(timezone.utc))
orders = self.algos[algo].run()
logger.info(
"algo: {}, orders: {}".format(
algo, [
o.__dict__ for o in orders]))
self._trade(orders)
counter += 1
spend_sec = time.time() - start_time
sleep_sec = self.run_freq_s - spend_sec
if sleep_sec < 0:
logger.warning("algo run time + trading time > run_freq_s: {} > {}, "
"try to use larger run_freq_s to fix it".format(spend_sec, self.run_freq_s))
sleep_sec = 0
else:
logger.info(
"algo run time + traing time: {}, sleep time: {}".format(spend_sec, sleep_sec))
time.sleep(sleep_sec)
def try_report(self, data):
if not self.is_send:
return
try:
is_ok, err_str = train_data_report(**data)
except Exception as err:
logger.warning(
'Exception: could not reach RemoteMonitor, exception:{}'.
format(err))
else:
if not is_ok:
logger.warning("report_train_data error:{}".format(err_str))
def getLast(self):
l = []
for id in [10, 11, 12, 13]:
url = "https://sportsbox.sa.gov.tw/material/list/" + str(id)
res = self.session.request("GET", url)
if res is None:
logger.warning(url + " is None")
return
soup = BeautifulSoup(res.content, "html.parser")
href = soup.select_one(
"div.itemBox.unit3_v.rounded.box_shadow.itemCard a")["href"]
l.append(int(href.replace("/material/detail/", "")))
return max(l)
async def job(self, id):
url = "https://sportsbox.sa.gov.tw/material/detail/" + str(id)
res = self.session.request("GET", url)
if res is None:
logger.warning(url + " is None")
return None
soup = BeautifulSoup(res.content, "html.parser")
title = soup.select_one("div.article_titleBox div.h4")
if title is None:
logger.warning(url + " is None because not find title")
return None
logger.info("GET data: " + url)
p = self.parser(soup, title.text, url)
return p
def stage3():
answer = input(
'\nAre you running in large scale manner (more than 5 machines in total)? [y/n] '
)
if answer == 'y':
answer = input(
'\nHave you followed the "A caveat for running in large scale" section in README? (i.e. you are using my public AMI) [y/n] '
)
if answer != 'y':
logger.warning('Please follow its instructions before proceeding.')
exit(0)
else:
cmds = [
'cd ~/heart_watch && pip3 install -r requirements.txt',
'cd ~/heart_watch && wget http://apache.mirrors.pair.com/kafka/2.2.0/kafka_2.12-2.2.0.tgz',
'cd ~/heart_watch && tar -xzf kafka_2.12-2.2.0.tgz && mv kafka_2.12-2.2.0 kafka',
]
else:
cmds = [
'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -',
'sudo add-apt-repository \'deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable\'',
'sudo add-apt-repository --remove -y ppa:andrei-pozolotin/maven3',
'sudo apt-get -y update',
'apt-cache policy docker-ce',
'sudo kill -9 $(ps aux | grep \'dpkg\' | awk \'{print $2}\')',
'sudo kill -9 $(ps aux | grep \'apt\' | awk \'{print $2}\')',
'sudo killall -r dpkg',
'sudo killall -r apt',
'sudo dpkg --configure -a',
'sudo apt-get install -y docker-ce python3-pip libpq-dev python-dev maven awscli',
'sudo usermod -aG docker ubuntu',
'cd ~/heart_watch && pip3 install -r requirements.txt',
'cd ~/ && wget http://apache.mirrors.pair.com/kafka/2.2.0/kafka_2.12-2.2.0.tgz',
'cd ~/ && tar -xzf kafka_2.12-2.2.0.tgz && mv kafka_2.12-2.2.0 kafka',
'unzip confluent-3.0.0-2.11.zip && mv confluent-3.0.0 confluent',
'cd ~/heart_watch && pip3 install -r requirements.txt',
]
for cmd in cmds:
parallel([
'peg sshcmd-cluster {} "{}"'.format(key, cmd) for key in CLUSTERS
])
logger.info("""
If you responded that you are NOT running in large scale manner, please manually ssh to each machine and run the following commands
sudo curl -L https://github.com/docker/compose/releases/download/1.24.1/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
sudo chmod +x /usr/local/bin/docker-compose
Then log out and log back again. Then make sure than you can run docker-compose. If you can't, please manually fix it.
""")
def request(self, method, url, data=None, delay=0, title=None):
for i in range(RETRY_CNT):
try:
if delay: time.sleep(delay)
return self.session.request(method,
url,
allow_redirects=False,
data=data,
timeout=self.timeout)
except (requests.HTTPError, requests.Timeout,
requests.ConnectionError) as e:
logger.warning('Warning: {0}, retrying({1}) ...'.format(
str(e), i))
pass
logger.error("can't get res: " + title)
return None
async def job(self, id):
url = "https://ed.arte.gov.tw/ch/content/m_design_content.aspx?AE_SNID=" + str(
id)
res = self.session.request("GET", url)
if res is None:
logger.warning(url + " is None")
return None
soup = BeautifulSoup(res.content, "html.parser")
title = soup.select_one(
"div.title_wrapper h3.animate.title-c1.title_icon_news").text
if len(title) == 0:
logger.warning(url + " is None because not find title")
return None
logger.info("GET data: " + url)
p = self.parser(soup, title, url)
return p
def _polygon_get_prices(self, symbols, freq, start_datetime, end_datetime):
df_dict = {}
for symbol in symbols:
try:
cur_df = self.polygon.get_historical_data(
symbol=symbol,
freq=freq,
start_date_str=start_datetime.strftime(
PolygonGateway.DATE_FMT),
end_date_str=end_datetime.strftime(
PolygonGateway.DATE_FMT),
unadjusted=False)
except PolygonRequestException as err:
logger.warning(
"polygon get error when load data, err:{}".format(err))
continue
gq_cur_df = data_polygon_to_goquant(cur_df)
df_dict[symbol] = gq_cur_df
return df_dict
async def job(self, id) -> Plan:
def func(type):
url = f"https://mlearn.moe.gov.tw/{type}/PartData?key={str(id)}"
soup = BeautifulSoup(
self.session.request("GET", url).content, "html.parser")
title = soup.select_one("div.container.mt-3 h2")
if title is None:
return url, soup, None
return url, soup, title
url, soup, title = func("TeachingPlan")
if title is None:
url, soup, title = func("TopicArticle")
if title is None:
logger.warning(url + " is None because not find title")
return None
logger.info("GET data: " + url)
p = self.parser(soup, title.text, url)
return p
def create(self):
db_connect_url_jdbc = 'jdbc:postgresql://{}:5432/postgres'.format(
getenv('DB_HOST'))
payload = {
'name': self.name,
'config': {
'connector.class':
'io.confluent.connect.jdbc.JdbcSourceConnector',
'connection.url': db_connect_url_jdbc,
'connection.user': '******',
'connection.password': '******',
'poll.interval.ms': self._poll_interval,
'numeric.mapping': 'best_fit',
'mode': 'bulk',
'transforms': 'createKey,extractInt',
'transforms.createKey.type':
'org.apache.kafka.connect.transforms.ValueToKey',
'transforms.createKey.fields': self._keyfield,
'transforms.extractInt.type':
'org.apache.kafka.connect.transforms.ExtractField$Key',
'transforms.extractInt.field': self._keyfield,
}
}
if self._query:
payload['config']['query'] = self._query
payload['config']['topic.prefix'] = self._topic.name
self._topic.create()
else:
logger.warning(
'"query" parameter is not specified. All messages will be sent to a single partition.'
)
payload['config']['table.whitelist'] = self._table_name
payload['config']['topic.prefix'] = util.naming.jdbc_topic_prefix()
if self._get():
self._delete()
self._create(payload)
def load_df(self, data_key):
if not self.check_data_key(data_key):
logger.warning(
"missing data key in cache, dt: {}".format(data_key))
return None
snapshot_path = self.get_data_snapshot_path(data_key)
data_files = [
join(snapshot_path, f) for f in os.listdir(snapshot_path)
if isfile(join(snapshot_path, f))
]
df = None
for filepath in data_files:
logger.debug("loading data from file: {}".format(filepath))
cur_df = pd.read_csv(filepath, index_col=False)
if df is None:
df = cur_df
else:
df = pd.concat([df, cur_df], sort=False)
if df is not None and df.shape[0] > 0:
df[DATA_DATETIME] = pd.to_datetime(df[DATA_DATETIME])
df = df.drop_duplicates()
df.set_index(DATA_DATETIME, inplace=True)
return df
def book_logger(title, msg):
open(LOG_FILE_NAME + ".book.log", 'a', encoding='utf-8').write(title + "," + msg + "\n")
logger.warning(title + "," + msg)
def on_capture_failure():
logger.warning('Task failed')
def shutdown(signal, frame):
logger.warning("Received signal %s: exiting", signal)
sys.exit(128 + signal)
def run(config):
rank = MPI.COMM_WORLD.Get_rank()
config.rank = rank
config.is_chef = rank == 0
config.seed = config.seed + rank
config.num_workers = MPI.COMM_WORLD.Get_size()
config.is_mpi = False if config.num_workers == 1 else True
if torch.get_num_threads() != 1:
fair_num_threads = max(
int(torch.get_num_threads() / MPI.COMM_WORLD.Get_size()), 1
)
torch.set_num_threads(fair_num_threads)
if config.is_chef:
logger.warning("Running a base worker.")
make_log_files(config)
else:
logger.warning("Running worker %d and disabling logger", config.rank)
logger.setLevel(CRITICAL)
if config.date is None:
now = datetime.now()
date = now.strftime("%m.%d")
else:
date = config.date
config.run_name = "rl.{}.{}.{}.{}".format(
config.env, date, config.prefix, config.seed - rank
)
if config.group is None:
config.group = "rl.{}.{}.{}".format(config.env, date, config.prefix)
config.log_dir = os.path.join(config.log_root_dir, config.run_name)
if config.is_train:
config.record_dir = os.path.join(config.log_dir, "video")
else:
config.record_dir = os.path.join(config.log_dir, "eval_video")
def shutdown(signal, frame):
logger.warning("Received signal %s: exiting", signal)
sys.exit(128 + signal)
signal.signal(signal.SIGHUP, shutdown)
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
# set global seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
os.environ["DISPLAY"] = ":1"
if config.gpu is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = "{}".format(config.gpu)
assert torch.cuda.is_available()
config.device = torch.device("cuda")
else:
config.device = torch.device("cpu")
# build a trainer
trainer = Trainer(config)
if config.is_train:
trainer.train()
logger.info("Finish training")
else:
trainer.evaluate()
logger.info("Finish evaluating")
def update(feedid: int) -> bool:
buildinfo = BuildInfo.return_item(feedid)
scheduler.add_job(update,
'interval',
args=[feedid],
id='feed{}'.format(feedid),
replace_existing=True,
minutes=int(buildinfo.check_interval))
rule = Ruleset.return_item(feedid) # type : Ruleset
_rmodified = False
# print("Auth:", rset.getauth, rset.auth1, rset.auth1param)
# if rset.getauth != '':
# _auth = {rset.auth1: rset.auth1param, rset.auth2: rset.auth2param,
# rset.auth3: rset.auth3param, rset.auth4: rset.auth4param,
# rset.auth5: rset.auth5param, }
#
# makeOpenerWithCookie(rset.getauth, _auth)
# else:
makeOpenerWithoutCookie(rule.listUrl)
# optional
encoding = rule.encoding
if encoding == '': encoding = 'utf-8'
if not rule.maxcheckpage or rule.maxcheckpage < 1:
logger.warning("*- rset.maxcheckpage is None, set default")
maxpage = 1
else:
maxpage = rule.maxcheckpage
if not rule.nextpagelink or not len(rule.nextpagelink):
maxpage = 1
logger.info("*- feed initializing....: feed_no. {}".format(feedid))
item_list, listpage = get_pageitems(rule, maxpage, encoding)
list_url = rule.listUrl
item_list = list(
map((lambda i: get_absolute_anchor_reference(list_url, i)), item_list))
if rule.encoding == '':
rule.encoding = listpage.original_encoding
_rmodified = True
# update_feed and update BuildInfo.lastBuildDate
if update_feed(feedid, rule, item_list):
buildinfo.lastBuildDate = time.strftime('%a, %d %b %Y %H:%M:%S GMT',
time.gmtime())
else:
logger.error("feed 생성에 실패하였음")
return False
# whether not updated feed(list not updated), change pub_date
buildinfo.pub_date = time.strftime('%a, %d %b %Y %H:%M:%S GMT',
time.gmtime())
if _rmodified:
# rule.save()
db_session.commit()
return True