def print_status(self):
"""
Prints the status out in a logger
:return:
"""
status_printer = Logger()
status_printer.log_success(self.get_status(), "JOB STATUS")
def test_print_sameline(capsys):
# Arrange
logger = Logger()
# Act
logger.print_sameline("hello")
logger.print_sameline("world")
captured = capsys.readouterr()
# Assert
assert captured.out == "\rhello\rworld"
def task_db_init():
log = Logger()
log.info('Connecting to database...')
result = app_load_db()
if result['exists']:
log.err('DB file already exists!')
return
dbm = AppGlobals.dbm
dbm.connect()
dbm.init_all()
# configuring base data
root_user = User(id=0,
name='admin',
passwd=usr_bake_password('222'),
org_id='0',
role=UserRoles.ADMIN.value,
is_admin=True,
prefs=UserSettings.serialize(UserSettings.default()))
user_prefs = UserSettings.default()
user_org_id = '15/0000001'
user_profile = user_prefs['userProfile']
user_profile['prettyName'] = 'Cristiano'
user_profile['orgID'] = user_org_id
example_student = User(id=1,
name='criss',
passwd=usr_bake_password('333'),
org_id=user_org_id,
role=UserRoles.STUDENT.value,
is_admin=False,
prefs=UserSettings.serialize(user_prefs))
dbm.session.add_all([root_user, example_student])
dbm.session.commit()
log.info('Done!')
class QuoteFeeder(object):
'''
classdocs
'''
logger = Logger.get_logger(__name__)
def __init__(self, folder):
if not os.path.exists(folder):
os.makedirs(folder)
self.folder = folder
def get_file_name_by_symbol(self, symbol):
return self.folder + os.path.sep + symbol
@abc.abstractmethod
def download_quotes(self, symbol, skip_existing):
return None
def fetch(self, symbol, skip_existing=True):
QuoteFeeder.logger.debug("Fetching quotes for symbol %s ..." % symbol)
file_name = self.get_file_name_by_symbol(symbol)
if skip_existing and os.path.exists(file_name):
QuoteFeeder.logger.info("Quotes for %s already exists in %s" %
(symbol, file_name))
else:
file_name = self.download_quotes(symbol, skip_existing)
if file_name != None:
QuoteFeeder.logger.info("Quotes for %s saved to file %s" %
(symbol, file_name))
else:
QuoteFeeder.logger.warning("Failed to fetch quotes for %s" %
symbol)
return file_name
def test_log_level_trace(capsys):
# Arrange
expected_included = ["trace", "debug", "success", "info", "warn", "error"]
logger = Logger(LogLevel.TRACE)
# Act
log_each(logger)
captured = capsys.readouterr()
# Assert
for msg in expected_included:
assert msg in captured.out
class QuoteUpdater(object):
logger = Logger.get_logger(__name__)
def __init__(self):
'''
Constructor
'''
self.quote_loader = QuoteLoader()
def connect(self):
self.quote_loader.connect()
def disconnect(self):
self.quote_loader.disconnect()
def update_quotes(self, symbols, start_date):
conn = http.client.HTTPConnection("ctxalgo.com")
url = "/api/ohlc/%s?start-date=%s&end-date=%s" % (
",".join(symbols), start_date, time.strftime("%Y-%m-%d"))
print(url)
conn.request("GET", url)
response_body = conn.getresponse().read().decode("utf-8")
loaded_json = json.loads(response_body)
eod_quotes = []
symbol_count = 0
for symbol in loaded_json:
symbol_count = symbol_count + 1
arrays = loaded_json[symbol]
for index, value in enumerate(arrays["dates"]):
date = arrays["dates"][index]
open_ = arrays["opens"][index]
high = arrays["highs"][index]
low = arrays["lows"][index]
close = arrays["closes"][index]
volume = arrays["volumes"][index]
amount = arrays["amounts"][index]
eod_quote = CtxEodQuote(symbol, date, open_, high, low, close,
volume, amount)
eod_quotes.append(eod_quote)
self.quote_loader.insert_eod_quote(eod_quote)
QuoteUpdater.logger.info(
"Updated %d quotes for %d symbols between %s and today" %
(len(eod_quotes), symbol_count, start_date))
def update_all_quotes(self, start_date):
stocks = Symbols.fetch_all_ctx_stocks()
chunks = [stocks[x:x + 50] for x in range(0, len(stocks), 50)]
for chunk in chunks:
symbols = map(lambda stock: stock.symbol, chunk)
self.update_quotes(symbols, start_date)
def test_log_level_error(capsys):
# Arrange
expected_included = ["error"]
expected_excluded = ["warn", "success", "info", "debug", "trace"]
logger = Logger(LogLevel.ERROR)
# Act
log_each(logger)
captured = capsys.readouterr()
# Assert
for msg in expected_included:
assert msg in captured.out
for msg in expected_excluded:
assert msg not in captured.out
def create_item(self, req: falcon.Request):
try:
result = self._players.insert_one(
req.context['body']
)
return str(result.inserted_id)
except (pymongoErrors.WriteError) as e:
Logger('app').error('Failed to parse date: ' + str(e))
self._handle_collection_invalid()
except (pymongoErrors.AutoReconnect,
pymongoErrors.ConnectionFailure,
pymongoErrors.NetworkTimeout):
self._handle_service_unavailable()
def get_list(self, _: falcon.Request) -> List[Dict]:
try:
# self._info("Fetching all players from datastore")
result = []
for player in self._players.find():
player['_id'] = str(player['_id'])
result.append(player)
return result
except (pymongoErrors.AutoReconnect,
pymongoErrors.ConnectionFailure,
pymongoErrors.NetworkTimeout) as e:
Logger('app').error('Failed to get players: ' + str(e))
self._handle_service_unavailable()
class QuoteLoader(object):
host = "server.jingyusoft.com"
keyspace = "stocks"
logger = Logger.get_logger(__name__)
def connect(self):
self.cluster = Cluster(contact_points=[QuoteLoader.host])
self.session = self.cluster.connect(QuoteLoader.keyspace)
def disconnect(self):
if self.cluster != None:
self.cluster.shutdown()
def insert_eod_quote(self, eod_quote):
cql = "insert into eod_quotes (symbol, date, open, high, low, close, volume) values ('%s', '%s', %.2f, %.2f, %.2f, %.2f, %d)" \
% (eod_quote.symbol, eod_quote.date.strftime("%Y-%m-%d"), eod_quote.open, eod_quote.high, eod_quote.low, eod_quote.close, eod_quote.volume)
self.session.execute(cql)
class CassandraSession(object):
'''
classdocs
'''
host = "server.jingyusoft.com"
keyspace = "stocks"
logger = Logger.get_logger(__name__)
def __init__(self):
'''
Constructor
'''
def connect(self):
self.cluster = Cluster(contact_points=[CassandraSession.host])
self.session = self.cluster.connect(CassandraSession.keyspace)
def disconnect(self):
self.cluster.shutdown()
def execute(self, query, parameters=None, trace=False):
return self.session.execute(query, parameters=parameters, trace=trace)
class MiioDeviceCollector(object):
labels = ['host', 'model', 'identifier']
def __init__(self, db: SQLAlchemy, prefix: str):
self.__prefix = prefix
self.__db = db
self.__log = Logger('mistats')
@staticmethod
def get_class(module, kls):
m = __import__(module)
for comp in kls.split('.'):
m = getattr(m, comp)
return m
def collect(self):
result = []
self.__db.connect()
objs = self.__db.session.query(Device).filter(
Device.enabled == True).all()
self.__db.close()
for device in objs:
info_class = self.get_class('miio', device.type)
if info_class is None:
continue
ip, _ = device.address.split(':', 1)
try:
dev_info = create_device(ip, info_class)
except DeviceError as e:
self.__log.error("error connecting to a device",
addr=ip,
device=info_class,
exc_info=e,
stack_info=True)
continue
except DeviceException as e:
self.__log.error("exception connecting to a device",
addr=ip,
device=info_class,
exc_info=e,
stack_info=True)
continue
status_method = getattr(dev_info, "status", None)
label_values = [device.address, device.type, device.identifier]
if callable(status_method):
try:
status = status_method()
except DeviceError as e:
self.__log.error("error querying device status",
addr=ip,
device=info_class,
exc_info=e,
stack_info=True)
continue
except DeviceException as e:
self.__log.error("exception querying device status",
addr=ip,
device=info_class,
exc_info=e,
stack_info=True)
continue
for name, value in getmembers(status.__class__):
if not isinstance(value, property):
continue
metric_value = getattr(status, name)
if metric_value is None:
continue
if isinstance(metric_value, Enum):
metric_value = metric_value.value
try:
metric_value = float(metric_value)
except ValueError:
continue
if value.__doc__ is not None:
doc = value.__doc__
else:
doc = ''
gauge = GaugeMetricFamily(self.__prefix + '_' +
name.lower(),
doc,
labels=self.labels)
gauge.add_metric(label_values, metric_value)
result.append(gauge)
return result
def on_get(self, req: falcon.Request, resp: falcon.Response) -> None:
data = self._controller.get_list(req)
Logger("app").info(str(data))
resp.body = PlayerSchema().dumps(data, many=True)
def log_each(logger: Logger):
logger.log_error('error msg')
logger.log_warn('warn msg')
logger.log_info('info blue msg')
logger.log_success('success msg')
logger.log_debug('debug msg')
logger.log_trace('trace msg')
from subprocess import CalledProcessError, run
from sys import exit as sys_exit
from time import sleep
from typing import List
from zlib import compress, decompress, error as CompressionException
from requests import Session, cookies, exceptions as RequestExceptions
# Internal imports
import common.api.endpoints as endpoints
from common.api.types import MasterInfo
from common.logging import Logger
from common.networking import get_ip_addr
from common.task import Task, TaskMessageType
from .heartbeat import Heartbeat
logger = Logger()
def run_shell_command(command):
"""
Execute a shell command outputing stdout/stderr to a result.txt file.
Returns the shell commands returncode.
:param command:
:return returncode:
"""
try:
output = run(command, check=True)
return output.returncode
def handle_exception(exception):
log = Logger('mistats')
log.error("exception occurred", exc_info=exception, stack_info=True)
raise hug.HTTPInternalServerError('Something broke', 'Something broke')
"""
Task Manager manages the Available, In Progress, and Completed Tasks for the Hyper Master
"""
from queue import SimpleQueue, Empty
from typing import List, Dict
from common.task import Task, TaskMessageType
from common.logging import Logger
from master.status_manager import StatusManager
logger = Logger()
class NoMoreTasks(Exception):
"""
Exception raised when there are no more tasks available or in progress
"""
class NoMoreAvailableTasks(Exception):
"""
Exception raised when there are no more tasks available
"""
class UnknownTaskMessage(Exception):
"""
Exception raised when the task manager encounters an unknown task message
"""
@author: Univer
'''
import sys, os, inspect
current_file = inspect.getfile(inspect.currentframe())
parent_dir = os.path.join(os.path.split(current_file)[0], os.path.pardir)
parent_folder = os.path.realpath(os.path.abspath(parent_dir))
if parent_folder not in sys.path:
sys.path.insert(0, parent_folder)
if __name__ == '__main__':
from symbols.symbols import Symbols
from common.cassandra import CassandraSession
from common.logging import Logger
logger = Logger.get_logger(__name__)
result = Symbols.fetch_all_ctx_stocks()
cassandra_session = CassandraSession()
try:
cassandra_session.connect()
for ctx_stock in result:
yahoo_symbol = Symbols.get_yahoo_symbol_from_ctx_symbol(ctx_stock.symbol)
Symbols.insert_symbol_mapping(cassandra_session, ctx_stock.symbol, yahoo_symbol, ctx_stock.name, ctx_stock.short_symbol)
logger.info("%s -> %s %s" % (ctx_stock.symbol, yahoo_symbol, ctx_stock.name))
Symbols.logger.info("Updated %d symbols from CoreTX" % len(result))
finally:
cassandra_session.disconnect()
class Symbols(object):
'''
classdocs
'''
logger = Logger.get_logger(__name__)
CtxStock = collections.namedtuple("CtxStock",
["symbol", "name", "short_symbol"],
verbose=False)
def __init__(self):
'''
Constructor
'''
@staticmethod
def fetch_all_ctx_stocks():
try:
Symbols.logger.info("Creating connection to ctxalgo.com ...")
conn = http.client.HTTPConnection("ctxalgo.com", timeout=5000)
Symbols.logger.info("Fetching stock list from /api/stocks ...")
conn.request("GET", "/api/stocks")
response = conn.getresponse().read().decode("utf-8")
Symbols.logger.debug("Response size is %d bytes" % len(response))
stocks = []
loaded_json = json.loads(response)
for symbol in loaded_json:
name = loaded_json[symbol]
stocks.append(Symbols.CtxStock(symbol, name, symbol[2:]))
return stocks
except:
return []
@staticmethod
def search(pattern):
symbol = Symbols.search_from_yahoo(pattern)
if symbol != None:
return [symbol]
else:
return Symbols.search_from_sina(pattern)
@staticmethod
def search_from_yahoo(pattern):
try:
conn = http.client.HTTPConnection("d.yimg.com", timeout=5000)
conn.request(
"GET",
"/aq/autoc?query=%s®ion=US&lang=en-US&callback=YAHOO.util.ScriptNodeDataSource.callbacks"
% pattern)
response = conn.getresponse().read().decode("utf-8")
matched = re.search('(?<="symbol":")[0-9]{6}.[A-Z]{2}', response)
if matched != None:
return matched.group(0)
else:
return None
except:
return None
@staticmethod
def get_yahoo_symbol_from_ctx_symbol(ctx_symbol):
test_yahoo_symbols = []
if ctx_symbol.startswith("sh"):
test_yahoo_symbols.append(ctx_symbol[2:] + ".SH")
elif ctx_symbol.startswith("sz"):
test_yahoo_symbols.append(ctx_symbol[2:] + ".SZ")
test_yahoo_symbols.append(ctx_symbol[2:] + ".SS")
for test_yahoo_symbol in test_yahoo_symbols:
try:
conn = http.client.HTTPConnection("ichart.finance.yahoo.com")
conn.request("GET", "/table.csv?s=%s" % test_yahoo_symbol)
if conn.getresponse().read(4).decode("utf-8") == "Date":
return test_yahoo_symbol
except:
return None
return None
@staticmethod
def search_from_sina(pattern):
symbols = []
try:
conn = http.client.HTTPConnection("suggest3.sinajs.cn",
timeout=10000)
conn.request("GET",
"/suggest/type=11,12,13,14,15&key=%s" % pattern)
response = conn.getresponse().read().decode("gbk")
matched = re.search("\"(.*)\"", response)
if matched != None:
for item in matched.group(1).split(";"):
code = item.split(",")[3]
conn = http.client.HTTPConnection("finance.sina.com.cn")
conn.request("GET",
"/realstock/company/%s/nc.shtml" % code)
response = conn.getresponse().read().decode("gbk")
search_pattern = "<span>(" + pattern + "."
start_index = response.index(search_pattern) + len(
"<span>(")
symbol = response[start_index:start_index + len(pattern) +
3]
symbols.append(symbol)
return symbols
except:
return []
@staticmethod
def insert_symbol_mapping(cassandra_session, ctx_symbol, yahoo_symbol,
name, short_symbol):
cassandra_session.execute(
"insert into symbols (ctx_symbol, yahoo_symbol, name, short_symbol, update_timestamp) values (%s, %s, %s, %s, dateof(now()))",
(ctx_symbol, yahoo_symbol, name, short_symbol))
def __init__(self, db: SQLAlchemy, prefix: str):
self.__prefix = prefix
self.__db = db
self.__log = Logger('mistats')
def run() -> falcon.API:
"""
:return: an initialized falcon.API
"""
Logger('app').info("timap service starting")
return initialize()
from config import DB, CONFIG
import routers
app = hug.API(__name__)
router = hug.route.API(app)
router.get('/metrics', output=hug.output_format.text,
api=app)(routers.generate_stats)
router.get('/devices', api=app)(routers.list_devices)
router.post('/devices/{device_id}/activate', api=app)(routers.acitvate_device)
router.post('/devices/discover', api=app)(routers.do_discovery)
initialize_logging(CONFIG.LOG_MODE, debug=CONFIG.DEBUG_MODE)
setup_access_log(app, Logger('access_log'))
DB.init_app(app, CONFIG.SQLALCHEMY_DATABASE_URI)
DB.create_models()
def signal_handler(signal, frame):
if ServiceDiscovery.service_browser.is_alive():
ServiceDiscovery.service_browser.cancel()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
@hug.exception(api=app)