def log2exception(code, sys_exc_info, message=None, die=False): """Log trace message to file and STDOUT, but don't die. Args: code: Message code sys_exc_info: Tuple from exception from sys.exc_info die: Die if True Returns: None """ # Initialize key variables (exc_type, exc_value, exc_traceback) = sys_exc_info log_message = ('''\ Bug: Exception Type:{}, Exception Instance: {}, Stack Trace Object: {}]\ '''.format(exc_type, exc_value, exc_traceback)) log2warning(code, log_message) if bool(message) is True: log2warning(code, message) # Write trace to log file from obya import Config config = Config() log_file = config.log_file with open(log_file, 'a+') as _fh: traceback.print_tb(exc_traceback, file=_fh) # Die if bool(die) is True: log2die(code, log_message)
def __init__(self, parent, child=None, config=None): """Initialize the class. Args: parent: Name of parent daemon child: Name of child daemon config: Config object Returns: None """ # Initialize key variables (Parent) if config is None: self.config = Config() else: self.config = config self.parent = parent self.pidfile_parent = config.pid_file(parent) self.lockfile_parent = config.lock_file(parent) # Initialize key variables (Child) if bool(child) is None: self._pidfile_child = None else: self._pidfile_child = config.pid_file(child)
def __init__(self, parent, child, app, config=None): """Initialize the class. Args: parent: Name of parent daemon child: Name of child daemon app: Flask App config: Config object Returns: None """ # Initialize key variables if config is None: _config = Config() else: _config = config # Apply inheritance Agent.__init__(self, parent, child=child, config=_config) self._app = app self._agent_api_variable = AgentAPIVariable( ip_bind_port=_config.ip_bind_port, ip_listen_address=_config.ip_listen_address)
def main(): """Process agent data. Args: None Returns: None """ # Initialize key variables use_mysql = True global POOL global URL pool_timeout = 30 pool_recycle = min(10, pool_timeout - 10) # Get configuration config = Config() # Create DB connection pool if use_mysql is True: URL = ('mysql+pymysql://{}:{}@{}/{}?charset=utf8mb4'.format( config.db_username, config.db_password, config.db_hostname, config.db_name)) # Fix for multiprocessing on pools. # _add_engine_pidguard(QueuePool) # Add MySQL to the pool db_engine = create_engine( URL, echo=False, echo_pool=False, encoding='utf8', poolclass=QueuePool, max_overflow=config.db_max_overflow, pool_size=config.db_pool_size, pool_pre_ping=True, pool_recycle=pool_recycle, pool_timeout=pool_timeout) # Fix for multiprocessing on engines. # _add_engine_pidguard(db_engine) # Ensure connections are disposed before sharing engine. db_engine.dispose() # Create database session object POOL = scoped_session( sessionmaker( autoflush=True, autocommit=False, bind=db_engine ) ) else: POOL = None
def __init__(self): """Initialize the class.""" # Application libraries from obya import Config # Define key variables app_name = 'obya' levels = { 'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL } # Get the logging directory config = Config() log_file = config.log_file config_log_level = config.log_level # Set logging level if config_log_level in levels: log_level = levels[config_log_level] else: log_level = levels['debug'] # create logger with app_name self.logger_file = logging.getLogger('{}_file'.format(app_name)) self.logger_stdout = logging.getLogger('{}_console'.format(app_name)) # Set logging levels to file and stdout self.logger_stdout.setLevel(log_level) self.logger_file.setLevel(log_level) # create file handler which logs even debug messages file_handler = logging.FileHandler(log_file) file_handler.setLevel(log_level) # create console handler with a higher log level stdout_handler = logging.StreamHandler() stdout_handler.setLevel(log_level) # create formatter and add it to the handlers formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') file_handler.setFormatter(formatter) stdout_handler.setFormatter(formatter) # add the handlers to the logger self.logger_file.addHandler(file_handler) self.logger_stdout.addHandler(stdout_handler)
def ingest(secondsago, verbose=False): """Ingest data from the API into the database. Args: secondsago: Amount of time to backfill verbose: Verbose output if true Returns: None """ # Initalize key variables timeframe = 14400 batch = 3000 config = Config() timestamps = [] # Calculate start, start stop = int(datetime.datetime.now().replace( tzinfo=datetime.timezone.utc).timestamp()) start = stop - secondsago # Get start and stop times items = list(range(start, stop, timeframe * batch)) items.append(stop) for index, value in enumerate(items[:-1]): timestamps.append({'start': value, 'stop': items[index + 1]}) # Get a list of pairs pairs = config.pairs # Ingest data _api = API() for pair in pairs: # Reporting if bool(verbose) is True: print('Processing {}'.format(pair)) # Process data for timestamp in timestamps: df_ = _api.historical(pair, timeframe, start=timestamp['start'], stop=timestamp['stop']) data.insert(pair, df_)
def main(): """Start the Gunicorn WSGI.""" # Initialize key variables config = Config() # Get PID filenename for Gunicorn agent_gunicorn = Agent(OBYA_WEBD_PROXY, config=config) # Get configuration agent_api = AgentAPI(OBYA_WEBD_NAME, OBYA_WEBD_PROXY, OBYA_WEBD, config=config) # Do control (API first, Gunicorn second) cli = AgentCLI() cli.control(agent_api) cli.control(agent_gunicorn)
def __init__(self): """Initialize the class. Args: None Returns: None """ # Initialize key variables self._http = urllib3.PoolManager() self._config = Config() self._base_url = 'https://{}/TradingAPI'.format( self._config.api_hostname) # Get the session key self.session_key = self._login()
def setup(): """Setup server. Args: None Returns: None """ # Initialize key variables use_mysql = True pool_size = 25 max_overflow = 25 # Get configuration config = Config() # Create DB connection pool if use_mysql is True: # Add MySQL to the pool engine = create_engine( URL, echo=True, encoding='utf8', max_overflow=max_overflow, pool_size=pool_size, pool_recycle=3600) # Try to create the database print('Attempting to create database tables') try: sql_string = ('''\ ALTER DATABASE {} CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci\ '''.format(config.db_name)) engine.execute(sql_string) except: log_message = '''\ Cannot connect to database {}. Verify database server is started. Verify \ database is created. Verify that the configured database authentication is \ correct.'''.format(config.db_name) log.log2die(1036, log_message) # Apply schemas print('Applying Schemas') BASE.metadata.create_all(engine)
def send(body, subject, attachments=None): """Read a configuration file. Args: body: Text for email body subject: Subject for email attachments: List of filepaths to attach Returns: success: True if succesful """ # Initialize key variables success = False config = Config() # Create SMTP TLS session client = smtplib.SMTP('smtp.gmail.com', 587) try: client.ehlo() except: _exception = sys.exc_info() log_message = 'Gmail Communication Failure' log.log2exception(1013, _exception, message=log_message) client.starttls() # Authentication try: client.login(config.smtp_user, config.smtp_pass) except: _exception = sys.exc_info() log_message = 'Gmail Authentication Failure' log.log2exception(1014, _exception, message=log_message) return success # Format message message = MIMEMultipart() message['Subject'] = subject message['From'] = config.email_from message['To'] = ', '.join(config.email_to) message.add_header('reply-to', config.email_from) html = ''' <html> <head></head> <body><font face="courier"> {} </font></body> </html> '''.format('<br>'.join(' '.join(body.split(' ')).split('\n'))) message.attach(MIMEText(html, 'html', 'UTF-8')) # Add attachment if required if bool(attachments) is True: if isinstance(attachments, list) is True: for attachment in attachments: part = MIMEApplication(open(attachment, 'rb').read()) part.add_header('Content-Disposition', 'attachment', filename=('{}'.format(attachment))) message.attach(part) # Send try: client.sendmail(config.email_from, config.email_to, message.as_string()) success = True except: _exception = sys.exc_info() log_message = 'Gmail Send Failure' log.log2exception(1015, _exception, message=log_message) return success finally: client.quit() return success