def __init__(self, path, symbol, message_queue):
self._message_queue = message_queue
# Configure logger
self._logger = logging.getLogger(f"{symbol}_MarketLogger")
self._logger.setLevel(logging.ERROR)
handler = logging.handlers.SysLogHandler(address='/dev/log')
self._logger.addHandler(handler)
self._plotProvider = PlotProvider()
# Path structure:
# path
# - genotick/
# - genotick.jar
# - <market_name>/
# - config.txt
# - data/
# - <market_symbol>.csv
# - robots/
# - robot files
self._path = os.path.abspath(path)
self._symbol = symbol
self._db = DatabaseManager()
self._genotick_path = fr"{self._path}/genotick/genotick.jar"
self._data_path = fr"{self._path}/{self._symbol}/data/{self._symbol}.csv"
self._reverse_data_path = fr"{self._path}/{self._symbol}/data/reverse_{self._symbol}.csv"
self._gen_config_path = fr"{self._path}/{self._symbol}/config.txt"
self._robots_path = fr"{self._path}/robots"
def main(argv):
usage = "usage: {} market_symbol".format(argv[0])
if len(argv) != 2:
print(usage)
sys.exit(1)
db = DatabaseManager()
pp = PlotProvider()
image = pp.get_market_24plot(db.get_24h_plot_data(argv[1]), argv[1][1:])
def _bot_job(self):
try:
db = DatabaseManager()
bot = Bot(self._bot_token)
chats = bot.get_chat_list()
for c in chats:
db.add_chat(c)
except Exception:
self._logger.exception("Failed to collect bot chats.")
def _daily_market_plot_job(self):
try:
db = DatabaseManager()
pp = PlotProvider()
markets = db.get_markets()
for m in markets:
data = db.get_24h_plot_data(m)
image = pp.get_market_24plot(data, m[1:])
self._message_queue.put({'type': 'image', 'data': image})
except Exception:
self._logger.exception("Failed to push daily market plots.")
def process_market_message(self):
try:
db = DatabaseManager()
bot = Bot(self._bot_token)
message = self._message_queue.get()
chats = db.get_chat_list()
if message["type"] == "text":
bot.send_text_message(message["data"], chats)
elif message["type"] == "image":
bot.send_image(message["data"], chats)
self._message_queue.task_done()
except Exception:
self._logger.exception(f"Failed to process market message.")
def __init__(self, config):
""" Initializes a bot manager with the given configuration.
Keyword arguments:
config -- The EngineConfig instance to use
"""
self.db_mgr = DatabaseManager(config)
def unsubscribe(self, email, token):
if email is None or email.isspace():
return False
email = email.strip()
if not validate_email(email):
print('유효하지 않은 이메일 주소', email)
return False
if token is None or token.isspace():
return False
if len(token) != 8:
print('유효하지 않은 토큰', token)
return False
id = int32(int(token, 16))
if id not in self.__subscriber_table:
print('구독중이 아닌 이메일 주소', email)
return False
DatabaseManager().remove_subscriber_by_id(id)
del self.__subscriber_table[id]
return True
def _predictions_job(self):
try:
db = DatabaseManager()
markets_list = db.get_markets()
# Create thread for each market
for m in markets_list:
if m in self._markets and self._markets[m].is_alive():
self._logger.error(
f"Thread for market {m} is still alive.")
continue
else:
t = threading.Thread(target=market_thread_func,
args=(m, self._path,
self._message_queue))
t.start()
self._markets[m] = t
except Exception:
self._logger.exception("Failed to start predictions job.")
class BotManager:
""" A class used to update information about online bots """
def __init__(self, config):
""" Initializes a bot manager with the given configuration.
Keyword arguments:
config -- The EngineConfig instance to use
"""
self.db_mgr = DatabaseManager(config)
def create_if_new(self, ip):
""" Determine if the host specified by ip exists in the bot status table,
and creates an initial record if not
"""
try:
results = self.db_mgr.exec_query('select * from bot_status where ip = ?', ip)
if len(results) == 0:
self.db_mgr.exec_cmd('''insert into bot_status (ip, last_startup_time, last_shutdown_time, last_activity_time)
values (?, ?, ?, ?)''', ip, None, None, None, commit=True)
except DBError, e:
print("Error occurred while querying bot status record exists for host {0}".format(ip))
raise e
def subscribe(self, email):
if email is None or email.isspace():
return False
email = email.strip()
if not validate_email(email):
print('유효하지 않은 이메일 주소', email)
return False
subscriber = Subscriber(email)
if subscriber.id in self.__subscriber_table:
print('이미 등록된 이메일 주소', email)
return False
DatabaseManager().add_subscriber(subscriber)
self.__subscriber_table[subscriber.id] = subscriber
return True
def __init__(self, path, bot_token):
self._bot_token = bot_token
self._logger = logging.getLogger('MarketManagerLogger')
self._logger.setLevel(logging.ERROR)
handler = logging.handlers.SysLogHandler(address='/dev/log')
self._logger.addHandler(handler)
self._db = DatabaseManager()
self._path = path
self._scheduler = BackgroundScheduler()
self._scheduler.add_job(self._daily_market_plot_job,
trigger='cron',
hour='0')
self._scheduler.add_job(self._predictions_job,
trigger='cron',
hour='*')
self._scheduler.add_job(self._bot_job, trigger='cron', minute='*')
self._markets = dict()
self._message_queue = queue.Queue()
def __init__(self):
handlers = [(r"/", WelcomeHandler),
(r"/login", LoginHandler),
(r"/logout", LogoutHandler),
(r"/list", ListHandler),
(r"/new", NewItemHandler),
(r"/action/(\w+)/(\w+)", ActionHandler),
]
settings = dict(
title="TODO LIST",
cookie_secret="SOME_RANDOM_VALUE",
template_path=os.path.join(os.path.dirname(__file__),
"templates"),
static_path=os.path.join(os.path.dirname(__file__),
"static"),
autoreload=True,
debug=True)
super(Application, self).__init__(handlers, **settings)
self.db = DatabaseManager()
def main():
"""
Downloads history from Mega-Sena website
and loads data into a MySQL database
"""
hd = HistoryDownloader('scraper-data')
hd.download_html()
hd.scrape_html()
hd.check_consistency()
hd.write_file('json')
hd.write_file('csv')
hd.create_db_csv_files()
hd.log_info()
dm = DatabaseManager(hd.draws_load_path, hd.winloc_load_path)
dm.create_tables()
dm.insert_data()
class TestManagerUpdates(unittest.TestCase):
""" A test case for the AccountManager updates """
def setUp(self):
""" Creates objects required by the test """
config = EngineConfig(os.path.join(os.path.dirname(__file__), 'weasel.conf'))
self.db_mgr = DatabaseManager(config)
self.clear_bot_status()
self.mgr = BotManager(config)
def tearDown(self):
self.db_mgr.shutdown()
def clear_bot_status(self):
""" Clears the contents of the bot status, command log, and command map tables"""
try:
self.db_mgr.exec_cmd('''delete from bot_status''', commit=True)
self.db_mgr.exec_cmd('''delete from command_map''', commit=True)
self.db_mgr.exec_cmd('''delete from command_log''', commit=True)
except DBError, e:
print("Error clearing database")
raise e
def setUp(self):
""" Creates objects required by the test """
config = EngineConfig(os.path.join(os.path.dirname(__file__), 'weasel.conf'))
self.db_mgr = DatabaseManager(config)
self.clear_bot_status()
self.mgr = BotManager(config)
def __update_subscriber_table(self):
subscribers = DatabaseManager().select_all_subscribers()
if len(subscribers) > 0:
self.__subscriber_table.clear()
for subscriber in subscribers:
self.__subscriber_table[subscriber.id] = subscriber
"""
Very simple straightforward testing of CRUD operations on a single item.
"""
from bson.objectid import ObjectId
from dbmanager import DatabaseManager
from taskitem import TaskItem
db = DatabaseManager()
taskid = ObjectId() # ID to use in test
testitem = TaskItem(_id=taskid,
user_id="123abc",
description="Test CRUD",
priority=1,
done=False)
def create():
db.create(testitem)
print("Item created")
read()
def read():
theitem = db.read_id(taskid)
print(theitem.document())
assert theitem.document() == testitem.document(), "Documents do not match"
def update():
testitem.done = True
print("Item updated")
"""
Very simple straightforward testing of CRUD operations on a single item.
"""
from bson.objectid import ObjectId
from dbmanager import DatabaseManager
from taskitem import TaskItem
db = DatabaseManager()
taskid = ObjectId() # ID to use in test
testitem = TaskItem(_id=taskid, user_id="123abc", description="Test CRUD",
priority=1, done=False)
def create():
db.create(testitem)
print("Item created")
read()
def read():
theitem = db.read_id(taskid)
print(theitem.document())
assert theitem.document() == testitem.document(), "Documents do not match"
def update():
testitem.done = True
print("Item updated")
db.update(testitem)
read()
def delete():
db.delete(testitem)
print("Item deleted")
from dbmanager import DatabaseManager
import pandas as pd
from timeseries import TimeSeries
from utils import Similarity
import inspect
import pika
import json
database_info = {
"url": "localhost",
"port": "27017",
"database": "Sibyl"
}
label = "analyzer"
db_manager = DatabaseManager(database_info, label)
global pewma
global cl
global ts
cl = StaticControlLimits()
pewma_model = Pewma()
ts = TimeSeries()
def entry_point(ch, method, properties, body):
topic = str(method).split('routing_key=')[-1].split("'")[0]
print(topic)
data = json.loads(body.decode("UTF-8"))
print(data)
data = transform_properties(data, topic)
class Market:
def __init__(self, path, symbol, message_queue):
self._message_queue = message_queue
# Configure logger
self._logger = logging.getLogger(f"{symbol}_MarketLogger")
self._logger.setLevel(logging.ERROR)
handler = logging.handlers.SysLogHandler(address='/dev/log')
self._logger.addHandler(handler)
self._plotProvider = PlotProvider()
# Path structure:
# path
# - genotick/
# - genotick.jar
# - <market_name>/
# - config.txt
# - data/
# - <market_symbol>.csv
# - robots/
# - robot files
self._path = os.path.abspath(path)
self._symbol = symbol
self._db = DatabaseManager()
self._genotick_path = fr"{self._path}/genotick/genotick.jar"
self._data_path = fr"{self._path}/{self._symbol}/data/{self._symbol}.csv"
self._reverse_data_path = fr"{self._path}/{self._symbol}/data/reverse_{self._symbol}.csv"
self._gen_config_path = fr"{self._path}/{self._symbol}/config.txt"
self._robots_path = fr"{self._path}/robots"
def genotick_predict_and_train(self):
try:
ts_prediction_start = self._db.get_last_predictions_ts(
self._symbol)
ts_history_start = self._db.get_last_history_ts(
self._symbol) * 1000
if (ts_prediction_start is None):
ts_prediction_start = ts_history_start
else:
ts_prediction_start *= 1000
ts_history_start += 60 * 60 * 1000
print("Collecting history data...")
history = bitfinex_api.append_1h_history(ts_history_start,
self._symbol,
self._data_path)
print("Adding data to database...")
self._db.append_market_history(history, self._symbol)
print("Configuring genotick for prediction...")
self._configure_genotick_prediction(ts_prediction_start)
print("Creating reverse data file...")
self._make_reverse_data_file()
print("Running genotick for prediction...")
predictions = self._parse_prediction_output(
self._genotick_predict())
if len(predictions) == 0:
self._logger.info(f"No predictions for market {self._symbol}")
return
print("Queuing predictions and plot to bot queue...")
self._enqueue_predictions(predictions)
#self._enqueue_market_plot()
print("Updating predictions in database...")
self._db.update_predictions(predictions, self._symbol)
print("Configuring genotick for training...")
self._configure_genotick_training(ts_history_start)
print("Running genotick for training...")
self._genotick_train()
except Exception:
self._logger.exception(
f"Failed to predict and train with genotick for market {self._symbol}"
)
def _get_custom_env(self):
result = os.environ.copy()
result["GENOTICK_LOG_FILE"] = f"{self._symbol}_genotick_log.txt"
return result
def _genotick_predict(self):
command = [
"java", "-jar", self._genotick_path,
f"input=file:{self._gen_config_path}"
]
cp = sp.run(command,
env=self._get_custom_env(),
universal_newlines=True,
stdout=sp.PIPE,
stderr=sp.PIPE)
if cp.returncode != 0:
raise RuntimeError(
f"Failed to run genotick in prediction mode for market {self._symbol}.",
cp.stdout, cp.stderr)
return cp.stdout
def _parse_prediction_output(self, output):
pattern = re.compile(
fr"^[\w\/\s]+\/{self._symbol}\.[\sa-z]+(\d+)[a-z\s]+\:\s(OUT|UP|DOWN)$",
re.MULTILINE)
items = pattern.findall(output)
# Add one hour for predictions timestamp
predictions = list()
for item in items:
predictions.append((int(item[0]) / 1000 + 60 * 60, item[1]))
return predictions
def _enqueue_predictions(self, predictions):
for p in predictions:
ts = datetime.datetime.utcfromtimestamp(int(
p[0])).strftime('%Y-%m-%d %H:%M:%S')
message = f"{ts} {self._symbol[1:]} {p[1]}"
self._message_queue.put({'type': 'text', 'data': message})
def _enqueue_market_plot(self):
data = self._db.get_24h_plot_data(self._symbol)
image = self._plotProvider.get_market_24plot(data, self._symbol[1:])
self._message_queue.put({'type': 'image', 'data': image})
def _remove_old_reverse_data_file(self):
command = ["rm", "-f", self._reverse_data_path]
cp = sp.run(command,
universal_newlines=True,
stdout=sp.PIPE,
stderr=sp.PIPE)
if cp.returncode != 0:
raise RuntimeError(
f"Failed to remove reverse data file for market {self._symbol}.",
cp.stdout, cp.stderr)
def _make_reverse_data_file(self):
self._remove_old_reverse_data_file()
command = [
"java", "-jar", self._genotick_path, f"reverse={self._data_path}"
]
cp = sp.run(command,
env=self._get_custom_env(),
universal_newlines=True,
stdout=sp.PIPE,
stderr=sp.PIPE)
if cp.returncode != 0:
raise RuntimeError(
f"Genotick failed to create reverse data file for market {self._symbol}. ",
cp.stdout, cp.stderr)
def _configure_genotick_prediction(self, start):
command = [
"sed", "-i", "-e",
r"s:\([#\s]*\)\(performTraining\s\+\)\(.\+\):\2false:", "-e",
fr"s:\([#\s]*\)\(startTimePoint\s\+\)\(.\+\):\2{start}:", "-e",
r"s/^[^#]*endTimePoint/#&/", self._gen_config_path
]
cp = sp.run(command,
universal_newlines=True,
stdout=sp.PIPE,
stderr=sp.PIPE)
if cp.returncode != 0:
raise RuntimeError(
f"Failed to configure genotick for prediction for market {self._symbol}.",
cp.stdout, cp.stderr)
def _configure_genotick_training(self, start):
command = [
"sed", "-i", "-e",
r"s:\([#\s]*\)\(performTraining\s\+\)\(.\+\):\2true:", "-e",
fr"s:\([#\s]*\)\(startTimePoint\s\+\)\(.\+\):\2{start}:", "-e",
r"s/^[^#]*endTimePoint/#&/", self._gen_config_path
]
cp = sp.run(command,
universal_newlines=True,
stdout=sp.PIPE,
stderr=sp.PIPE)
if cp.returncode != 0:
raise RuntimeError(
f"Failed to configure genotick for training for market {self._symbol}.",
cp.stdout, cp.stderr)
def _genotick_train(self):
command = [
"java", "-jar", self._genotick_path,
f"input=file:{self._gen_config_path}"
]
with sp.Popen(command,
env=self._get_custom_env(),
universal_newlines=True,
stdout=sp.PIPE,
stderr=sp.PIPE) as proc:
pid = proc.pid
try:
outs, errs = proc.communicate(timeout=(45 * 60))
except TimeoutError:
proc.kill()
outs, errs = proc.communicate()
raise RuntimeError(
f"Failed to run genotick in training mode for market {self._symbol}. Error: {outs}. {errs}"
)
newRobotsPath = f"savedPopulation_{pid}"
#print(fr"New population path for market {self._symbol} is {newRobotsPath}")
command = [
"rm", "-f", "-r", self._robots_path, "&&", "mv", newRobotsPath,
self._robots_path
]
cp = sp.run(command,
universal_newlines=True,
stdout=sp.PIPE,
stderr=sp.PIPE)
if cp.returncode != 0:
raise RuntimeError(
f"Failed to move new robots for market {self._symbol}.",
cp.stdout, cp.stderr)