def log2exception(code, sys_exc_info, message=None, die=False):
"""Log trace message to file and STDOUT, but don't die.
Args:
code: Message code
sys_exc_info: Tuple from exception from sys.exc_info
die: Die if True
Returns:
None
"""
# Initialize key variables
(exc_type, exc_value, exc_traceback) = sys_exc_info
log_message = ('''\
Bug: Exception Type:{}, Exception Instance: {}, Stack Trace Object: {}]\
'''.format(exc_type, exc_value, exc_traceback))
log2warning(code, log_message)
if bool(message) is True:
log2warning(code, message)
# Write trace to log file
from obya import Config
config = Config()
log_file = config.log_file
with open(log_file, 'a+') as _fh:
traceback.print_tb(exc_traceback, file=_fh)
# Die
if bool(die) is True:
log2die(code, log_message)
def __init__(self, parent, child=None, config=None):
"""Initialize the class.
Args:
parent: Name of parent daemon
child: Name of child daemon
config: Config object
Returns:
None
"""
# Initialize key variables (Parent)
if config is None:
self.config = Config()
else:
self.config = config
self.parent = parent
self.pidfile_parent = config.pid_file(parent)
self.lockfile_parent = config.lock_file(parent)
# Initialize key variables (Child)
if bool(child) is None:
self._pidfile_child = None
else:
self._pidfile_child = config.pid_file(child)
def __init__(self, parent, child, app, config=None):
"""Initialize the class.
Args:
parent: Name of parent daemon
child: Name of child daemon
app: Flask App
config: Config object
Returns:
None
"""
# Initialize key variables
if config is None:
_config = Config()
else:
_config = config
# Apply inheritance
Agent.__init__(self, parent, child=child, config=_config)
self._app = app
self._agent_api_variable = AgentAPIVariable(
ip_bind_port=_config.ip_bind_port,
ip_listen_address=_config.ip_listen_address)
def main():
"""Process agent data.
Args:
None
Returns:
None
"""
# Initialize key variables
use_mysql = True
global POOL
global URL
pool_timeout = 30
pool_recycle = min(10, pool_timeout - 10)
# Get configuration
config = Config()
# Create DB connection pool
if use_mysql is True:
URL = ('mysql+pymysql://{}:{}@{}/{}?charset=utf8mb4'.format(
config.db_username, config.db_password,
config.db_hostname, config.db_name))
# Fix for multiprocessing on pools.
# _add_engine_pidguard(QueuePool)
# Add MySQL to the pool
db_engine = create_engine(
URL,
echo=False,
echo_pool=False,
encoding='utf8',
poolclass=QueuePool,
max_overflow=config.db_max_overflow,
pool_size=config.db_pool_size,
pool_pre_ping=True,
pool_recycle=pool_recycle,
pool_timeout=pool_timeout)
# Fix for multiprocessing on engines.
# _add_engine_pidguard(db_engine)
# Ensure connections are disposed before sharing engine.
db_engine.dispose()
# Create database session object
POOL = scoped_session(
sessionmaker(
autoflush=True,
autocommit=False,
bind=db_engine
)
)
else:
POOL = None
def __init__(self):
"""Initialize the class."""
# Application libraries
from obya import Config
# Define key variables
app_name = 'obya'
levels = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warning': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL
}
# Get the logging directory
config = Config()
log_file = config.log_file
config_log_level = config.log_level
# Set logging level
if config_log_level in levels:
log_level = levels[config_log_level]
else:
log_level = levels['debug']
# create logger with app_name
self.logger_file = logging.getLogger('{}_file'.format(app_name))
self.logger_stdout = logging.getLogger('{}_console'.format(app_name))
# Set logging levels to file and stdout
self.logger_stdout.setLevel(log_level)
self.logger_file.setLevel(log_level)
# create file handler which logs even debug messages
file_handler = logging.FileHandler(log_file)
file_handler.setLevel(log_level)
# create console handler with a higher log level
stdout_handler = logging.StreamHandler()
stdout_handler.setLevel(log_level)
# create formatter and add it to the handlers
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
file_handler.setFormatter(formatter)
stdout_handler.setFormatter(formatter)
# add the handlers to the logger
self.logger_file.addHandler(file_handler)
self.logger_stdout.addHandler(stdout_handler)
def ingest(secondsago, verbose=False):
"""Ingest data from the API into the database.
Args:
secondsago: Amount of time to backfill
verbose: Verbose output if true
Returns:
None
"""
# Initalize key variables
timeframe = 14400
batch = 3000
config = Config()
timestamps = []
# Calculate start, start
stop = int(datetime.datetime.now().replace(
tzinfo=datetime.timezone.utc).timestamp())
start = stop - secondsago
# Get start and stop times
items = list(range(start, stop, timeframe * batch))
items.append(stop)
for index, value in enumerate(items[:-1]):
timestamps.append({'start': value, 'stop': items[index + 1]})
# Get a list of pairs
pairs = config.pairs
# Ingest data
_api = API()
for pair in pairs:
# Reporting
if bool(verbose) is True:
print('Processing {}'.format(pair))
# Process data
for timestamp in timestamps:
df_ = _api.historical(pair,
timeframe,
start=timestamp['start'],
stop=timestamp['stop'])
data.insert(pair, df_)
def main():
"""Start the Gunicorn WSGI."""
# Initialize key variables
config = Config()
# Get PID filenename for Gunicorn
agent_gunicorn = Agent(OBYA_WEBD_PROXY, config=config)
# Get configuration
agent_api = AgentAPI(OBYA_WEBD_NAME,
OBYA_WEBD_PROXY,
OBYA_WEBD,
config=config)
# Do control (API first, Gunicorn second)
cli = AgentCLI()
cli.control(agent_api)
cli.control(agent_gunicorn)
def __init__(self):
"""Initialize the class.
Args:
None
Returns:
None
"""
# Initialize key variables
self._http = urllib3.PoolManager()
self._config = Config()
self._base_url = 'https://{}/TradingAPI'.format(
self._config.api_hostname)
# Get the session key
self.session_key = self._login()
def setup():
"""Setup server.
Args:
None
Returns:
None
"""
# Initialize key variables
use_mysql = True
pool_size = 25
max_overflow = 25
# Get configuration
config = Config()
# Create DB connection pool
if use_mysql is True:
# Add MySQL to the pool
engine = create_engine(
URL, echo=True,
encoding='utf8',
max_overflow=max_overflow,
pool_size=pool_size, pool_recycle=3600)
# Try to create the database
print('Attempting to create database tables')
try:
sql_string = ('''\
ALTER DATABASE {} CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\
'''.format(config.db_name))
engine.execute(sql_string)
except:
log_message = '''\
Cannot connect to database {}. Verify database server is started. Verify \
database is created. Verify that the configured database authentication is \
correct.'''.format(config.db_name)
log.log2die(1036, log_message)
# Apply schemas
print('Applying Schemas')
BASE.metadata.create_all(engine)
def send(body, subject, attachments=None):
"""Read a configuration file.
Args:
body: Text for email body
subject: Subject for email
attachments: List of filepaths to attach
Returns:
success: True if succesful
"""
# Initialize key variables
success = False
config = Config()
# Create SMTP TLS session
client = smtplib.SMTP('smtp.gmail.com', 587)
try:
client.ehlo()
except:
_exception = sys.exc_info()
log_message = 'Gmail Communication Failure'
log.log2exception(1013, _exception, message=log_message)
client.starttls()
# Authentication
try:
client.login(config.smtp_user, config.smtp_pass)
except:
_exception = sys.exc_info()
log_message = 'Gmail Authentication Failure'
log.log2exception(1014, _exception, message=log_message)
return success
# Format message
message = MIMEMultipart()
message['Subject'] = subject
message['From'] = config.email_from
message['To'] = ', '.join(config.email_to)
message.add_header('reply-to', config.email_from)
html = '''
<html>
<head></head>
<body><font face="courier">
{}
</font></body>
</html>
'''.format('<br>'.join(' '.join(body.split(' ')).split('\n')))
message.attach(MIMEText(html, 'html', 'UTF-8'))
# Add attachment if required
if bool(attachments) is True:
if isinstance(attachments, list) is True:
for attachment in attachments:
part = MIMEApplication(open(attachment, 'rb').read())
part.add_header('Content-Disposition',
'attachment',
filename=('{}'.format(attachment)))
message.attach(part)
# Send
try:
client.sendmail(config.email_from, config.email_to,
message.as_string())
success = True
except:
_exception = sys.exc_info()
log_message = 'Gmail Send Failure'
log.log2exception(1015, _exception, message=log_message)
return success
finally:
client.quit()
return success