def test_Configuration_read_test(self): f = open(self.confPath, 'w', encoding="utf-8") f.write(self.testConf) f.close() conf = Configuration(self.confPath) self.assertEqual(conf.get_conf("Client", "database"), "../db") self.assertEqual(conf.get_conf("Client", "frontend-port"), 5000) os.remove(self.confPath)
def run_from_config(config_ff, threadID): # Load settings and create the config object config = Configuration(config_ff) settings = config.settings # Set up logger logs.configure_logging(settings['log_level'], settings['log_mode']) logger = structlog.get_logger() logger.info(" %s", threadID) # Configure and run configured behaviour. exchange_interface = ExchangeInterface(config.exchanges) notifier = Notifier(config.notifiers, config) behaviour = Behaviour(config, exchange_interface, notifier) while True: if settings['run_on_start']: start = time.time() behaviour.run(settings['market_pairs'], settings['output_mode']) end = time.time() dif = end - start wait_for = settings['update_interval'] - int(dif) logger.info("Sleeping for %s seconds", wait_for) time.sleep(wait_for) else: logger.info("Run on start not enabled waiting for %s seconds", settings['wait_and_run']) time.sleep(settings['wait_and_run'])
def portfolio_crypto_fiat(portfolio): conf = Configuration() currency = conf.settings_main['currency'] rates = exchangerates.get_data() try: exchange_rate = rates['rates'][currency] except: key_list = ['USD'] for key in rates['rates'].keys(): key_list.append(key) logger.warn('{} not found in exchangerates, possible symbols are: {}'.format(currency, key_list)) raise logger.info("Calculating crypto portfolio value in {}".format(currency)) openmarketcap_data = openmarketcap.get_data() openmc_df = openmarketcap.parse_data(openmarketcap_data) portfolio_fiat = portfolio_crypto_format(portfolio) portfolio_fiat = portfolio_fiat.copy() for token in list(portfolio_fiat.index.values): ls = openmc_df.loc[openmc_df['symbol'] == token.upper()] if ls.empty: logger.warn("{} not found in openmarketcap api data".format(token.upper())) usd_price = float(ls.iloc[0]['price_usd']) portfolio_fiat.loc[token, :] *= usd_price * exchange_rate portfolio_fiat = portfolio_fiat.round(2) return portfolio_fiat, currency
def config(): conf = Configuration() settings = conf.settings discordbot = conf.discordbot message_templater = conf.message dockerimage = conf.docker return settings, discordbot, message_templater, dockerimage
def main(): config = Configuration() handle = dataHandler() alerts = Alerts() for k, v in config.modclass.items(): parseStrats = StrategyParse(config.Strats[k], v) parseStrats.sock.start()
def main(): """Initializes the application """ # Load settings and create the config object config = Configuration() settings = config.settings # Set up logger logs.configure_logging(settings['log_level'], settings['log_mode']) logger = structlog.get_logger() # Configure and run configured behaviour. exchange_interface = ExchangeInterface(config.exchanges) if settings['market_pairs']: market_pairs = settings['market_pairs'] logger.info("Found configured markets: %s", market_pairs) market_data = exchange_interface.get_exchange_markets( markets=market_pairs) else: logger.info("No configured markets, using all available on exchange.") market_data = exchange_interface.get_exchange_markets() # notifier = Notifier(config.notifiers, market_data) thread_list = [] for exchange in market_data: num = 1 for chunk in split_market_data(market_data[exchange]): market_data_chunk = dict() market_data_chunk[exchange] = { key: market_data[exchange][key] for key in chunk } notifier = Notifier(config.notifiers, config.indicators, market_data_chunk) behaviour = Behaviour(config, exchange_interface, notifier) workerName = "Worker-{}".format(num) worker = AnalysisWorker(workerName, behaviour, notifier, market_data_chunk, settings, logger) thread_list.append(worker) worker.daemon = True worker.start() time.sleep(60) num += 1 logger.info('All workers are running!') for worker in thread_list: worker.join()
def setup(): conf = Configuration('..') prices_manager = Prices('prices', conf['se']) today = date.today() range_dates = ('2017-01-01', today.isoformat()) ckp_manager = CheckpointManager('checkpoint') data = ckp_manager.load_base() if data is not None: print('last checkpoint loaded') symbols = data['symbols'] all_data = data['all_data'] else: blacklist = set(['CAPU', 'PESA', 'PSUR', 'POLL']) symbols = conf.symbols() all_data = load_all_data(prices_manager, blacklist, symbols, range_dates) print('calculating returns') for data in all_data.values(): # TODO there must be a better way to do this returns = pd.DataFrame(data['Adj Close']).apply( lambda x: np.log(x) - np.log(x.shift())) data['Daily Return'] = returns state = { 'all_data': all_data, 'symbols': symbols, } ckp_manager.save_base(state) print("using dates [%s - %s]" % range_dates) # strategy = StrategyFindBestMA(ckp_manager) strategy = StrategyRandom01(ckp_manager) print(f'using strategy {type(strategy).__name__}') strategy.load(all_data.keys(), all_data) return (today, range_dates, conf, prices_manager, all_data, ckp_manager, strategy)
def __init__(self, exchange): self.ex = exchange self.exchange = getattr(ccxt, exchange)({ "enableRateLimit": True }) config = Configuration() self.indicator_conf = config.indicators self.informant_conf = config.informants self.crossover_conf = config.crossovers # TODO: analyzer self.strategy_analyzer = StrategyAnalyzer()
def main(): """Initializes the application """ # Load settings and create the config object config = Configuration() settings = config.settings # Set up logger logs.configure_logging(settings['log_level'], settings['log_mode']) logger = structlog.get_logger() while True: logger.info("Waking up just to do nothing!") logger.info("Sleeping for %s seconds", settings['update_interval']) time.sleep(settings['update_interval'])
def __init__(self, config: Configuration, path, mission_name): self._logger = logging.getLogger(__name__) """ Get Satellite catalog number from tle to be used as name for log file. """ try: tle_req = requests.get(config.get_conf("Client", "tle-url"), timeout=5) if tle_req.status_code == 200: mission_name = tle_req.text.split("\n")[1][2:7] except: self.log.warning( "Connection to tle endpoint failed. Using default packet log file name ({})" .format(mission_name)) if not os.path.isdir(path): os.mkdir(path) logfile = "{}/{}_packets.log".format(path, mission_name) self.file_logger = logging.FileHandler(logfile)
def test_Configuration_write_test(self): f = open(self.confPath, 'w', encoding="utf-8") f.write(self.testConf) f.close() conf = Configuration(self.confPath) conf.set_conf("Client", "frontend-port", 4700) conf.set_conf("Client", "static-files-path", "../static") f = open(self.confPath, 'r', encoding="utf-8") content = f.read() f.close() self.assertNotEqual(content, self.writtenConf) os.remove(self.confPath)
def main(): """Initializes the application """ # Load settings and create the config object config = Configuration() settings = config.settings # Set up logger logs.configure_logging(settings['log_level'], settings['log_mode']) logger = structlog.get_logger() # Configure and run configured behaviour. exchange_interface = ExchangeInterface(config.exchanges) notifier = Notifier(config.notifiers) behaviour = Behaviour(config, exchange_interface, notifier) while True: behaviour.run(settings['market_pairs'], settings['output_mode']) logger.info("Sleeping for %s seconds", settings['update_interval']) time.sleep(settings['update_interval'])
def main(argv): """ Main loop function. """ """ Parse command line options """ opts, args = getopt(argv, "vc:") conf_path = None for opt, arg in opts: if opt == "-c": conf_path = arg if conf_path is None: """ Default conf path """ conf_path = "../configuration.ini" """ Create the configuration object """ conf = Configuration(conf_path) """ Set up logging """ if not conf.get_conf("Client", "debug-log"): logging.basicConfig(level=logging.INFO) else: logging.basicConfig(level=logging.DEBUG) _logger = logging.getLogger(__name__) if not os.path.isdir("../logs"): os.mkdir("../logs") formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') handler = logging.FileHandler("../logs/system_logs.log") handler.setFormatter(formatter) logging.getLogger('').addHandler(handler) _logger.info("Using configuration from: %s", conf_path) """ Create the database object """ db_loc = os.path.join(util.get_root(), conf.get_conf("Client", "database")) database = TelemetryDB(db_loc) database.init_db() """ Update grafana and kaitai configurations """ if conf.get_conf("Client", "automatic-updating"): Updater(conf).checkForUpdates() """ Build the other components. """ ax_listener = AXListener(conf) sids_relay = SIDSRelay(conf, database) telemetry_listener = TelemetryListener(database) file_logger = FileLogger(conf, conf.get_conf("Client", "logs"), "log") """ Create the flask app and start it in a forked process. """ port = conf.get_conf("Client", "frontend-port") """ Set the handler for SIGTERM, so we can exit a bit more gracefully. """ signal.signal(signal.SIGTERM, terminate_handler) """ Hook the callbacks to the ax_listener. """ ax_listener.add_callback(database.insert_ax_frame) ax_listener.add_callback(sids_relay.relay) ax_listener.add_callback(file_logger.log_ax_frame) ax_listener.add_callback(telemetry_listener.receive) tnc_pool = TNCPool(conf, ax_listener) tnc_pool.connect_main_tnc() api_app = api.create_app(conf, tnc_pool, sids_relay) """ We set the daemon option to True, so that the client will quit once the other threads have finished because we don't have a good way of stopping the Flask app properly. """ api_thread = Thread(target=api_app.run, kwargs={"port": port}, daemon=True) api_thread.start() _logger.info("For the GUI open localhost:{}".format(port)) try: """ On windows, the KeyboardInterrupt doesn't break the join. """ if platform.system() == "Windows": while api_thread.isAlive: api_thread.join(2) else: api_thread.join() except (KeyboardInterrupt, SystemExit): pass finally: tnc_pool.cleanup()
import matplotlib.pyplot as plt from logs import configure_logging from conf import Configuration from backtesting.chart import Chart from backtesting.strategy import BacktestingStrategy # Load settings and create the config object config = Configuration() settings = config.get_settings() # Set up logger configure_logging(settings['loglevel'], settings['log_mode']) # Sample coin pairs to backtest with coins = [ "ETH/BTC", "LTC/BTC", "XMR/BTC", "OMG/BTC", "XRP/BTC", "SC/BTC", "XEM/BTC", "DASH/BTC", "LSK/BTC", "GNT/BTC", "VTC/BTC", "ETC/BTC", "STRAT/BTC", "DGB/BTC" ] """ A Backtesting engine """ class Backtester(object): def __init__(self, coin_pair, period_length, exchange, capital,
import concurrent.futures import logs import structlog import copy #To store config per user users_config = dict() users_market_data = dict() users_exchanges = dict() users_indicators = dict() #New analysis results updated each 5min new_results = dict() # Load settings and create the config object config = Configuration() settings = config.settings # Set up logger logs.configure_logging(settings['log_level'], settings['log_mode']) logger = structlog.get_logger() update_interval = ceil(settings['update_interval'] / 60) logger.info('udate interval %d ', update_interval) config_indicators = config.indicators # Configure and run configured behaviour. exchange_interface = ExchangeInterface(config.exchanges) if settings['market_pairs']:
from flask import Flask from pyfladesk import init_gui from conf import Configuration import logs import structlog UPLOAD_FOLDER = '/home/ikwtif/uploads' app = Flask(__name__) app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER conf = Configuration() settings_log = conf.settings_log settings_main = conf.settings_main logs.configure_logging(settings_log['log_level'], settings_log['log_mode']) from __init__ import * if __name__ == "__main__": if settings_main['app']: init_gui(app, width=1000, height=1000) else: app.run(debug=True, host='0.0.0.0', port=8080, passthrough_errors=True)
@app.errorhandler(404) def not_found(e): return send_file(os.path.join(static_folder, "index.html")) @app.route("/api/static/<path:path>") def send_static(path): return send_from_directory("static", path) @app.route("/api/conf", methods=["POST"]) def post_set_conf(): some_json = request.get_json() try: for i in some_json: for j in some_json[i]: config.set_conf(section=i, element=j, value=some_json[i][j]) except: _, error, _ = sys.exc_info() return jsonify({"Error": "{err}".format(err=error)}), 500 return jsonify(some_json), 200 return app if __name__ == "__main__": CONF_PATH = os.path.join(util.get_root(), "configuration.ini") conf = Configuration(CONF_PATH) STATIC_PATH = os.path.join(util.get_root(), conf.get_conf("Client", "static-files-path")) APP = create_app(conf, STATIC_PATH, None) APP.run(debug=True)
def get_configuration(self, file = 'guanxi.build.conf.yml', guanxiConf = 'guanxi_configuration'): conf = Configuration(file) conf.set_conf(conf.get_configuration(guanxiConf)) return conf
from flask import Flask from conf import Configuration app = Flask(__name__) app.config.from_object(Configuration())
def create_app(config: Configuration, tnc_pool: TNCPool, sids_relay: SIDSRelay) -> Flask: """ Creates a flask app for the api. """ log = logging.getLogger(__name__) static_folder = os.path.join(util.get_root(), config.get_conf("Client", "static-files-path")) app = Flask(__name__, static_url_path="", static_folder=static_folder) CORS(app) if not config.get_conf("Client", "debug-log"): server_log = logging.getLogger("werkzeug") server_log.setLevel(logging.WARN) # swagger specific swagger_url = "/api/docs" api_url = "/api/static/swagger.yaml" swaggerui_blueprint = get_swaggerui_blueprint( swagger_url, api_url, config={ "app_name": "Estcube 2 Telemetry API" } ) app.register_blueprint(swaggerui_blueprint, url_prefix=swagger_url) # end swagger specific @app.route("/api/sids/status", methods=["GET"]) def get_sids_status(): return jsonify(sids_relay.get_status()), 200 @app.route("/api/sids/toggle", methods=["POST"]) def toggle_relay(): response_json = request.get_json() current_relay_status = response_json["Mission Control"]["relay-enabled"] config.set_conf(section="Mission Control", element="relay-enabled", value=current_relay_status) if current_relay_status: threading.Thread(target=sids_relay.relay_unrelayed_packets, daemon=True).start() return response_json, 200 @app.route("/api/tnc/<name>/status", methods=["GET"]) def get_tnc_connection_check(name: str): if tnc_pool is None: return jsonify({"error": "TNC Pool is not defined."}), 500 res = tnc_pool.check_tnc(name) return jsonify({"name": name, "status": res.name}), 200 @app.route("/api/tnc/Main/start", methods=["POST"]) def post_tnc_main_start(): if tnc_pool is None: return jsonify({"error": "TNC Pool is not defined."}), 500 tnc_pool.connect_main_tnc() return "", 204 @app.route("/api/tnc/<name>/stop", methods=["POST"]) def post_tnc_connection_stop(name: str): if tnc_pool is None: return jsonify({"error": "TNC Pool is not defined."}), 500 tnc_pool.stop_tnc(name) return "", 204 @app.route("/api/conf", methods=["GET"]) def getconf(): """ Returns the whole current configuration object. """ res = config.get_all_conf() return jsonify(res) @app.route("/api/conf/constraints", methods=["GET"]) def get_constraints(): """ Returns all of the constraints for the configuration. """ constrs = config.get_constraints() return jsonify(constrs) @app.route("/api/conf/full", methods=["GET"]) def get_full_conf(): res = config.get_conf_with_constraints() return res @app.route("/", methods=["GET"]) def get_index(): return send_file(os.path.join(static_folder, "index.html")) @app.errorhandler(404) def not_found(e): return send_file(os.path.join(static_folder, "index.html")) @app.route("/api/static/<path:path>") def send_static(path): return send_from_directory("static", path) @app.route("/api/conf", methods=["POST"]) def post_set_conf(): some_json = request.get_json() try: for i in some_json: for j in some_json[i]: config.set_conf(section=i, element=j, value=some_json[i][j]) except: _, error, _ = sys.exc_info() return jsonify({"Error": "{err}".format(err=error)}), 500 return jsonify(some_json), 200 return app
class UnitTest(unittest.TestCase): testConf = """\ [Client] database=../db frontend-port=5000 """ writtenConf = """\ [Client] database=../db frontend-port=5000 static-files-path=../static """ confPath = os.path.join(os.path.dirname(__file__), "__test__", "conf.ini") dbPath = os.path.join(os.path.dirname(__file__), "__test__", "test.db") testConfiguration = Configuration(confPath) axPacket = b"""~\x8a\xa6j\x8a@@`\x8a\xa6j\x8a\x86@a\x03\xf0\x00\x00\x00\x01\x00q?GmN2dzpYLYwjaf\ RIg30bY;BJ:K/JyOUu1tVqkch\\TN>dx~""" infoArr = bytearray.fromhex("""0000000100713f476d4e32647a70594c59776a61665249673\ 33062593b424a3a4b2f4a794f5575317456716b63685c544e3e""".replace(" ", "")) beaconDataICPFrame = b'\x9a\x92\x86\xa8\xa4\x98`\x8a\xa6\xa8\x86\xaa\x84w\x03\xf0\x01\x074\xf7\x9a\xbaZ\x03${]\xa8_,\x01\x0f\x00\x00\x00\t\x05\x00\x00\x00\x9a\x00\x00\x00\x03\xd2\x04\x00\x00\xea\x04\x00\x00\xb0\x04b\xbd\x0190_\xd6\xff\xff\xff\xff\x0f\xfe\xc8\xfe\xc8d\x08\x0c4:\x05\x05\xe7x~' telemetryAXFrame = AXFrame(None, None, None, None, None, beaconDataICPFrame, None, datetime(2019, 11, 21)) def test_telemetry_decoder(self): f = open(os.path.join(os.path.dirname(__file__), "..", "spec", "telemetry.json"), "r", encoding="utf-8") conf = f.read() f.close() database = TelemetryDB(self.dbPath) database.init_db() listener = TelemetryListener(database) database.conn.close() os.remove(self.dbPath) # Tests if a packet is correctly stored in AXFrame by sending a packet and checking if current date is saved def test_AXListener_addresses(self): listener = AXListener(self.testConfiguration) def assertDestAndSrc(frame: AXFrame): self.assertEqual(frame.dest, "ES5E ", "Frame destination is decoded incorrectly.") self.assertEqual(frame.source, "ES5EC ", "Frame source is decoded incorrectly.") listener.add_callback(assertDestAndSrc) listener.receive(bytearray(self.axPacket)) def test_AXListener_control(self): listener = AXListener(self.testConfiguration) def assertControlByte(frame: AXFrame): self.assertEqual(frame.ctrl, 3) listener.add_callback(assertControlByte) listener.receive(bytearray(self.axPacket)) def test_AXListener_pid(self): listener = AXListener(self.testConfiguration) def assertPID(frame: AXFrame): self.assertEqual(frame.pid, 240) listener.add_callback(assertPID) listener.receive(bytearray(self.axPacket)) def test_AXListener_info(self): listener = AXListener(self.testConfiguration) def assertControlByte(frame: AXFrame): self.assertEqual(frame.info, self.infoArr, "\nframe: {}\ntest: {}".format( frame.info, self.infoArr)) listener.add_callback(assertControlByte) listener.receive(bytearray(self.axPacket)) def test_Configuration_read_test(self): f = open(self.confPath, 'w', encoding="utf-8") f.write(self.testConf) f.close() conf = Configuration(self.confPath) self.assertEqual(conf.get_conf("Client", "database"), "../db") self.assertEqual(conf.get_conf("Client", "frontend-port"), 5000) os.remove(self.confPath) # Tests if set_conf method changes the values in the conf file. def test_Configuration_write_test(self): f = open(self.confPath, 'w', encoding="utf-8") f.write(self.testConf) f.close() conf = Configuration(self.confPath) conf.set_conf("Client", "frontend-port", 4700) conf.set_conf("Client", "static-files-path", "../static") f = open(self.confPath, 'r', encoding="utf-8") content = f.read() f.close() self.assertNotEqual(content, self.writtenConf) os.remove(self.confPath) # Tests if an AXFame is correctly stored in the DB. def test_DB_test(self): database = TelemetryDB(self.dbPath) database.init_db() ts = datetime.now() frame = AXFrame(None, None, None, None, None, None, bytearray(self.axPacket), ts) database.insert_ax_frame(frame) # Placeholder assert since querying DB is not implemented cur = database.conn.cursor() cur.execute("select frame_timestamp, frame_data from ax_frame order by frame_timestamp desc limit 1") time, data = cur.fetchone() self.assertEqual(time, ts.isoformat()) self.assertEqual(data, self.axPacket) database.conn.close() os.remove(self.dbPath)
row_date = from_req_str(datum[0]) dates.append(row_date) cols = [float(x.replace(',', '')) for x in datum[1:]] prices_data.append(cols) prices = pd.DataFrame(data=prices_data, index=dates, columns=[ 'Open', 'Max', 'Min', 'Close', 'Adj Close', 'Volumen monto', 'Volumen nominal' ]) prices.index.name = 'Date' if cur_prices is None: cur_prices = prices else: cur_prices = cur_prices.append(prices, sort=True) cur_prices.drop_duplicates(keep='last', inplace=True) prices_manager.save(symbol, cur_prices) if __name__ == '__main__': conf = Configuration('..') downloader = Downloader('cache') prices_manager = Prices('prices', conf['se']) _setup_symbols() _update_prices()
import tornado.web import aiodocker import discord from discord.ext import commands import logging # TODO - fix docker setup # # monkey patch du to 3.8 breaking change for tornado if sys.platform == 'win32': asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) # { # loading config logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.INFO) configuration = Configuration(logging) # } # logging setup loglevel = configuration.settings.get('loglevel') if loglevel: if loglevel == 'info': logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.INFO) logging.info('logging set to info') elif loglevel == 'debug': logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.DEBUG) logging.info('logging set to debug') # {
directory = osPath.dirname(osPath.abspath(__file__)) # get base directory length = len(directory) etcPath = directory[0:length - len('sys')] + 'etc' basePath = directory[0:length - len('utils/sys')] connectionString = f'sqlite:///{basePath}/utils/sys/data.db' # set db connection string # load data with open(etcPath + '/schema.json', 'r') as s: schema = Schema.fromJSON(loads(s.read())) with open(etcPath + '/dialog.json', 'r') as d: dialog = Dialog.fromJSON(loads(d.read())) with open(etcPath + '/config.json', 'r') as c: config = Configuration.fromJSON(loads(c.read())) # add required packages to sys.path for pkg in config['server']['required']: if not pkg in path: path.insert(0, basePath + pkg) # prepare log from log import Log log = Log.fromConfig(config['server']['log']) # prepare flask, sqlalchemy, marshmallow, and wtforms from app import application from flask_sqlalchemy import SQLAlchemy from flask_marshmallow import Marshmallow application.config['SQLALCHEMY_DATABASE_URI'] = connectionString