def __init__(self, debug_dir, debug_level, debug_name): self.rtree = radix.Radix() # Logger for the pipeline logger.setup_logger('fwtable', debug_dir + '/' + debug_name, debug_level) self.log = logging.getLogger('fwtable')
def __init__(self, log_dir, log_level, registers, window_size=10, bin_time=0.05): # Logger for the throughput monitor self.log_dir = log_dir logger.setup_logger('throughput', log_dir+'/throughput.log', level=log_level) self.log = logging.getLogger('throughput') self.registers = registers self.window_size = window_size self.bin_time = bin_time
def do_visualization( cfg, model, backward_model, dataloader, device, ): logger = setup_logger('balad-mobile.visualization', False) logger.info("Start visualizing") create_path(cfg.OUTPUT.VIS_DIR) for iteration, (images, targets, paths) in tqdm(enumerate(dataloader)): indices = targets.view(-1).nonzero().type(torch.LongTensor) images = images.to(device) targets = targets.to(device) predictions, activations = model(images, targets) backward_segmentations = backward_model(activations) images, steering_commands, backward_segmentations, indices = \ images.cpu(), targets.cpu(), backward_segmentations.cpu(), indices.cpu() for i in indices: image, image_mask = images[i, :, :, :], backward_segmentations[i, :, :, :] path = cfg.OUTPUT.VIS_DIR + "/" + paths[i] worker(image, image_mask, path)
def do_train(cfg, model, dataloader_train, dataloader_evaluation, optimizer, device): # set mode to training for model (matters for Dropout, BatchNorm, etc.) model.train() # get the trainer logger and visdom visdom = VisdomLogger(cfg.LOG.PLOT.DISPLAY_PORT) visdom.register_keys(['loss']) logger = setup_logger('balad-mobile.train', False) logger.info("Start training") output_dir = os.path.join( cfg.LOG.PATH, 'run_{}'.format(datetime.now().strftime("%Y-%m-%d_%H:%M:%S"))) os.makedirs(output_dir) # start the training loop for epoch in range(cfg.SOLVER.EPOCHS): for iteration, (images, steering_commands, _) in enumerate(dataloader_train): images = images.to(device) steering_commands = steering_commands.to(device) predictions, loss = model(images, steering_commands) optimizer.zero_grad() loss.backward() optimizer.step() if iteration % cfg.LOG.PERIOD == 0: visdom.update({'loss': [loss.item()]}) logger.info("LOSS: \t{}".format(loss)) if iteration % cfg.LOG.PLOT.ITER_PERIOD == 0: visdom.do_plotting() step = epoch * len(dataloader_train) + iteration if step % cfg.LOG.WEIGHTS_SAVE_PERIOD == 0 and iteration: torch.save( model.state_dict(), os.path.join(output_dir, 'weights_{}.pth'.format(str(step)))) do_evaluation(cfg, model, dataloader_evaluation, device) torch.save(model.state_dict(), os.path.join(output_dir, 'weights_final.pth'))
def main(): parser = argparse.ArgumentParser(description="PyTorch Self-driving Car Training and Inference.") parser.add_argument( "--config-file", default="", metavar="file", help="path to config file", type=str, ) parser.add_argument( "--mode", default="test", metavar="mode", help="'train' or 'test'", type=str, ) parser.add_argument( "opts", help="Modify config options using the command-line", default=None, nargs=argparse.REMAINDER, ) args = parser.parse_args() # build the config cfg = get_cfg_defaults() # cfg.merge_from_file(args.config_file) # cfg.merge_from_list(args.opts) cfg.freeze() # setup the logger if not os.path.isdir(cfg.OUTPUT.DIR): os.mkdir(cfg.OUTPUT.DIR) logger = setup_logger("balad-mobile.train", cfg.OUTPUT.DIR, '{0:%Y-%m-%d %H:%M:%S}_log'.format(datetime.now())) logger.info(args) logger.info("Running with config:\n{}".format(cfg)) # TRAIN train(cfg) # Visualize visualization(cfg)
def make_logging(self): # Logger for the pipeline logger.setup_logger('p4_to_controller', self.log_dir+'/p4_to_controller_'+ \ str(self.sw_name)+'.log', level=logging.INFO) self.log = logging.getLogger('p4_to_controller') # Logger for the sliding window logger.setup_logger('p4_to_controller_sw', self.log_dir+'/p4_to_controller_'+ \ str(self.sw_name)+'_sw.log', level=logging.INFO) self.log_sw = logging.getLogger('p4_to_controller_sw') # Logger for the rerouting logger.setup_logger('p4_to_controller_rerouting', self.log_dir+'/p4_to_controller_'+ \ str(self.sw_name)+'_rerouting.log', level=logging.INFO) self.log_rerouting = logging.getLogger('p4_to_controller_rerouting') # Logger for the Flow Selector logger.setup_logger('p4_to_controller_fs', self.log_dir+'/p4_to_controller_'+ \ str(self.sw_name)+'_fs.log', level=logging.INFO) self.log_fs = logging.getLogger('p4_to_controller_fs')
def do_evaluation( cfg, model, dataloader, device, verbose=False, ): logger = setup_logger('balad-mobile.evaluation', False) logger.info("Start evaluating") loss_records = [] for iteration, (images, steering_commands, _) in tqdm(enumerate(dataloader)): images = images.to(device) steering_commands = steering_commands.to(device) predictions, loss = model(images, steering_commands) loss_records.append(loss.item()) if verbose: logger.info("LOSS: \t{}".format(loss)) logger.info('LOSS EVALUATION: {}'.format(np.mean(loss_records)))
def main(): setup_logger() logger = logging.getLogger() def cmc_examples(): with open("bin/keys/cmc_k.txt", "r") as f: cmc_key = f.read() cmc = CryptoWrapper(api="CMC", api_key=cmc_key, cache_expire=240, max_retries=1) cmc_wrapper = cmc.wrapper # Get list of available endpoints (functions) # logger.info(cmc_wrapper.__getfunctions__()) resp = cmc_wrapper.global_aggregate_metrics_latest_GET() logger.info(resp["data"]["quote"]["USD"]["total_market_cap"]) resp = cmc_wrapper.cryptocurrency_info_GET(symbol="BTC") logger.info(resp["data"]["BTC"]["date_added"]) def cryptocompare_examples(): with open("bin/keys/cryptocompare_k.txt", "r") as f: cryptocompare_key = f.read() cryptocompare = CryptoWrapper(api="CryptoCompare", api_key=cryptocompare_key) cryptocompare_wrapper = cryptocompare.wrapper # Get list of available endpoints (functions) # logger.info(cryptocompare_wrapper.__getfunctions__()) resp = cryptocompare_wrapper.price_GET(fsym="BTC", tsyms="USD,JPY,EUR") logger.info(resp) resp = cryptocompare_wrapper.historical_daily_ohlcv_GET(fsym="BTC", tsym="USD", limit=1) logger.info(resp) def bitmex_examples(): # with open("bin/keys/bitmex_k.txt", "r") as f: # bitmex_key = f.read() # with open("bin/keys/bitmex_s.txt", "r") as f: # bitmex_secret = f.read() bitmex = CryptoWrapper( api="BitMEX", # api_key=bitmex_key, # api_secret=bitmex_secret, cache_expire=0, max_retries=2) bitmex_wrapper = bitmex.wrapper # To alternate between mainnet & testnet: bitmex_wrapper.BASE_URL = "https://www.bitmex.com/api/v1" # bitmex_wrapper.BASE_URL = "https://testnet.bitmex.com/api/v1" def example_1(): resp = bitmex_wrapper.announcement_GET(columns=["title", "date"]) logger.info(resp[0]) def example_2(): resp = bitmex_wrapper.order_bulk_POST(orders=json.dumps([{ "symbol": "XBTUSD", "orderQty": 250, "price": 1000 }, { "symbol": "XBTUSD", "orderQty": 500, "price": 2500 }]), ) logger.info(resp) def example_3(): resp = bitmex_wrapper.chat_GET(count=2) logger.info(resp[0]["date"]) # Get list of available endpoints (functions) # logger.info(bitmex_wrapper.__getfunctions__()) example_1() # example_2() # example_3() def binance_examples(): # with open("bin/keys/binance_k.txt", "r") as f: # binance_key = f.read() # with open("bin/keys/binance_s.txt", "r") as f: # binance_secret = f.read() binance = CryptoWrapper( api="Binance", # api_key=binance_key, # api_secret=binance_secret, max_retries=2) binance_wrapper = binance.wrapper def example_1(): resp = binance_wrapper.exchange_information_GET() logger.info(resp) def example_2(): resp = binance_wrapper.order_test_POST( symbol="LTCBTC", side="BUY", type="LIMIT", timeInForce="GTC", quantity=10, price=0.009, recvWindow=5000, timestamp=int(time() * 1000 - 2000)) logger.info(resp) def example_3(): resp = binance_wrapper.user_wallet_deposit_address_GET( asset="BTC", recvWindow=5000, timestamp=int(time() * 1000 - 2000)) logger.info(resp) # Get list of available endpoints (functions) # logger.info(binance_wrapper.__getfunctions__()) example_1() # example_2() # example_3() def binance_dex_examples(): binance_dex = CryptoWrapper(api="BinanceDEX", max_retries=2) binance_dex_wrapper = binance_dex.wrapper # Get list of available endpoints (functions) # logger.info(binance_dex_wrapper.__getfunctions__()) resp = binance_dex_wrapper.fees_GET() logger.info(resp) resp = binance_dex_wrapper.tokens_GET() logger.info(resp) def bitfinex_examples(): bitfinex = CryptoWrapper(api="Bitfinex") bitfinex_wrapper = bitfinex.wrapper # Get list of available endpoints (functions) # logger.info(bitfinex_wrapper.__getfunctions__()) resp = bitfinex_wrapper.platform_status_GET() logger.info(resp) resp = bitfinex_wrapper.tickers_GET(symbols="tBTCUSD,tLTCUSD,fUSD") logger.info(resp) def deribit_examples(): # with open("bin/keys/deribit_k.txt", "r") as f: # deribit_key = f.read() # with open("bin/keys/deribit_s.txt", "r") as f: # deribit_secret = f.read() deribit = CryptoWrapper( api="Deribit", # api_key=deribit_key, # api_secret=deribit_secret, cache_expire=0, max_retries=2) deribit_wrapper = deribit.wrapper # To alternate between mainnet & testnet: deribit_wrapper.BASE_URL = "https://www.deribit.com/api/v2" # deribit_wrapper.BASE_URL = "https://test.deribit.com/api/v2" def example_1(): resp = deribit_wrapper.get_time_GET() logger.info(resp) def example_2(): resp = deribit_wrapper.get_contract_size_GET( instrument_name="BTC-PERPETUAL") logger.info(resp) def example_3(): resp = deribit_wrapper.order_buy_GET( instrument_name="BTC-PERPETUAL", amount=500, type="limit", label="test", price=1000) logger.info(resp) # Get list of available endpoints (functions) # logger.info(deribit_wrapper.__getfunctions__()) example_1() # example_2() # example_3() try: cmc_examples() # cryptocompare_examples() # bitmex_examples() # binance_examples() # binance_dex_examples() # bitfinex_examples() # deribit_examples() except Exception as e: logger.info(f"Exception: {e}")
from util.discord.channel import ChannelUtil from util.env import Env from util.logger import setup_logger from version import __version__ import asyncio import discord import logging from rpc.client import RPCClient from tasks.transaction_queue import TransactionQueue # Configuration config = Config.instance() # Setup logger setup_logger(config.log_file, log_level=logging.DEBUG if config.debug else logging.INFO) logger = logging.getLogger() client = Bot(command_prefix=config.command_prefix) client.remove_command('help') ### Bot events @client.event async def on_ready(): logger.info(f"Starting Graham v{__version__}") logger.info(f"Discord.py version {discord.__version__}") logger.info(f"Bot name: {client.user.name}") logger.info(f"Bot Discord ID: {client.user.id}") await client.change_presence(activity=discord.Game(config.playing_status))
required=True) parser.add_argument('--threshold', type=int, default=31, help='Threshold used to decide when to fast reroute') args = parser.parse_args() port = args.port log_dir = args.log_dir log_level = args.log_level topo_db = args.topo_db routing_file = args.routing_file threshold = args.threshold # Logger for the controller logger.setup_logger('controller', log_dir + '/controller.log', level=log_level) log = logging.getLogger('controller') log.info(str(port)+'\t'+str(log_dir)+'\t'+str(log_level)+'\t'+str(routing_file)+ \ '\t'+str(threshold)) # Read the topology topo = Topology(db=topo_db) mapping_dic = {} tmp = list(topo.get_hosts()) + list(topo.get_p4switches()) mapping_dic = {k: v for v, k in enumerate(tmp)} log.info(str(mapping_dic)) """ This function adds an entry in a match+action table of the switch """
def __init__(self, port, log_dir, log_level, window_size, nbprefixes, \ nbflows_prefix, eviction_timeout, seed): # Logger for the pipeline logger.setup_logger('pipeline', log_dir + '/pipeline.log', level=log_level) self.log = logging.getLogger('pipeline') self.log.log(20, str(port)+'\t'+str(log_dir)+'\t'+str(log_level)+'\t'+ \ str(window_size)+'\t'+str(nbprefixes)+'\t'+str(nbflows_prefix)+'\t'+ \ '\t'+str(eviction_timeout)+'\t'+str(seed)) self.ip_controller = 'localhost' self.port_controller = port self.seed = seed # Dictionnary with all the forwarding table self.fwtables = {} self.fwtables['meta_fwtable'] = FWTable(log_dir, log_level, 'meta_fwtable.log') # Dictionnary with all the registers array self.registers = {} self.registers['flowselector_key'] = [0] * ( nbflows_prefix * nbprefixes ) # *100 to make sure there is no overflow self.registers['flowselector_ts'] = [0] * (nbflows_prefix * nbprefixes) self.registers['flowselector_nep'] = [0] * ( nbflows_prefix * nbprefixes) # nep for Next Expected Packet self.registers['flowselector_last_ret'] = [0] * ( nbflows_prefix * nbprefixes) # Timestamp self.registers['flowselector_5tuple'] = [''] * ( nbflows_prefix * nbprefixes) # Just used in the python implem # Registers used for the sliding window self.registers['sw'] = [] for _ in xrange(0, nbprefixes): self.registers['sw'] += [0] * window_size self.registers['sw_index'] = [0] * nbprefixes self.registers['sw_time'] = [0] * nbprefixes self.registers['sw_sum'] = [0] * nbprefixes # Registers used for the throughput sliding window self.registers['sw_throughput'] = [] for _ in xrange(0, nbprefixes): self.registers['sw_throughput'] += [0] * window_size self.registers['sw_index_throughput'] = [0] * nbprefixes self.registers['sw_time_throughput'] = [0] * nbprefixes self.registers['sw_sum1_throughput'] = [0] * nbprefixes self.registers['sw_sum2_throughput'] = [0] * nbprefixes self.registers['threshold_registers'] = [50] * nbprefixes self.registers['next_hops_index'] = [0] * nbprefixes self.registers['next_hops_port'] = [2, 3, 4] * nbprefixes # This is the FlowSelector, use to keep track of a defined number of # active flows per prefix self.flowselector = FlowSelector(log_dir, 20, self.registers, 32, \ nbflows_prefix, eviction_timeout, self.seed) self.throughput = ThroughputMonitor(log_dir, 20, self.registers) # Socket used to communicate with the controller self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_address = (self.ip_controller, self.port_controller) while True: status = self.sock.connect_ex(server_address) if status == 0: print 'Connected!' break else: print 'Could not connect, retry in 2 seconds..' time.sleep(2) fcntl.fcntl(self.sock, fcntl.F_SETFL, os.O_NONBLOCK) self.data = '' # Read flow table rules from the controller until the table is fully populated self.ready = False while not self.ready: self.read_controller() pass
"""Module to be run first time to set up the database * Drops all tables if the exist and creates them again. * Populates responses from Gamepedia * Populates heroes from Gamepedia and Dota 2 subreddit CSS. """ from parsers import css_parser, wiki_parser from util.database.database import db_api __author__ = 'MePsyDuck' from util.logger import setup_logger def first_run(): db_api.drop_all_tables() db_api.create_all_tables() wiki_parser.populate_responses() css_parser.populate_heroes() if __name__ == '__main__': setup_logger() first_run()
type=str, default='log', help='Log Directory', required=False) args = parser.parse_args() dst_ip = args.dst_ip src_ports = args.src_ports dst_ports = args.dst_ports ipd = args.ipd duration = args.duration log_dir = args.log_dir process_list = [] logger.setup_logger('traffic_generation', log_dir + '/traffic_generation.log', level=logging.INFO) log = logging.getLogger('traffic_generation') for src_port, dst_port in zip(range(int(src_ports.split(',')[0]), int(src_ports.split(',')[1])), \ range(int(dst_ports.split(',')[0]), int(dst_ports.split(',')[1]))): flow_template = { "dst": dst_ip, "dport": dst_port, "sport": src_port, "ipd": ipd, "duration": duration } process = multiprocessing.Process(target=sendFlowTCP, kwargs=flow_template)
from weixin.weixin import server from util.logger import setup_logger import logging setup_logger('weixin.log') _logger = logging.getLogger(__name__) while True: try: server() except Exception as e: _logger.exception(e)
def main(args): utils.init_distributed_mode(args) logger = setup_logger(output=args.output_dir, distributed_rank=dist.get_rank(), name="DETR", phase="train" if not args.eval else "eval") logger.info(args) # logger.info("git:\n {}\n".format(utils.get_sha())) if args.frozen_weights is not None: assert args.masks, "Frozen training is meant for segmentation only" device = torch.device(args.device) # fix the seed for reproducibility seed = args.seed + utils.get_rank() torch.manual_seed(seed) np.random.seed(seed) random.seed(seed) model, criterion, postprocessors = build_model(args) logger.info(model) model.to(device) model_without_ddp = model if args.distributed: model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.gpu]) model_without_ddp = model.module n_parameters = sum(p.numel() for p in model.parameters() if p.requires_grad) logger.info('number of params: {}'.format(n_parameters)) param_dicts = [ { "params": [ p for n, p in model_without_ddp.named_parameters() if "backbone" not in n and p.requires_grad ] }, { "params": [ p for n, p in model_without_ddp.named_parameters() if "backbone" in n and p.requires_grad ], "lr": args.lr_backbone, }, ] optimizer = torch.optim.AdamW(param_dicts, lr=args.lr, weight_decay=args.weight_decay) lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.lr_drop) dataset_train = build_dataset(image_set='train', args=args) dataset_val = build_dataset(image_set='val', args=args) if args.distributed: sampler_train = DistributedSampler(dataset_train) sampler_val = DistributedSampler(dataset_val, shuffle=False) else: sampler_train = torch.utils.data.RandomSampler(dataset_train) sampler_val = torch.utils.data.SequentialSampler(dataset_val) batch_sampler_train = torch.utils.data.BatchSampler(sampler_train, args.batch_size, drop_last=True) data_loader_train = DataLoader(dataset_train, batch_sampler=batch_sampler_train, collate_fn=utils.collate_fn, num_workers=args.num_workers) data_loader_val = DataLoader(dataset_val, args.batch_size, sampler=sampler_val, drop_last=False, collate_fn=utils.collate_fn, num_workers=args.num_workers) if args.dataset_file == "coco_panoptic": # We also evaluate AP during panoptic training, on original coco DS coco_val = datasets.coco.build("val", args) base_ds = get_coco_api_from_dataset(coco_val) else: base_ds = get_coco_api_from_dataset(dataset_val) if args.frozen_weights is not None: checkpoint = torch.load(args.frozen_weights, map_location='cpu') model_without_ddp.detr.load_state_dict(checkpoint['model']) output_dir = Path(args.output_dir) if args.resume: if args.resume == 'auto': ckpt = os.path.join(args.output_dir, 'checkpoint.pth') if os.path.isfile(ckpt): checkpoint = torch.load(os.path.join(args.output_dir, 'checkpoint.pth'), map_location='cpu') msg = model_without_ddp.load_state_dict(checkpoint['model'], strict=False) logger.info('Missing keys: {}'.format(msg.missing_keys)) if not args.eval and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint: optimizer.load_state_dict(checkpoint['optimizer']) lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) args.start_epoch = checkpoint['epoch'] + 1 else: if args.resume.startswith('https'): checkpoint = torch.hub.load_state_dict_from_url( args.resume, map_location='cpu', check_hash=True) else: checkpoint = torch.load(args.resume, map_location='cpu') model_without_ddp.load_state_dict(checkpoint['model']) if not args.eval and 'optimizer' in checkpoint and 'lr_scheduler' in checkpoint and 'epoch' in checkpoint: optimizer.load_state_dict(checkpoint['optimizer']) lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) args.start_epoch = checkpoint['epoch'] + 1 if args.eval: test_stats, coco_evaluator = evaluate(model, criterion, postprocessors, data_loader_val, base_ds, device, args.output_dir) if args.output_dir: utils.save_on_master(coco_evaluator.coco_eval["bbox"].eval, output_dir / "eval.pth") return logger.info("Start training") start_time = time.time() for epoch in range(args.start_epoch, args.epochs): if args.distributed: sampler_train.set_epoch(epoch) train_stats = train_one_epoch(model, criterion, data_loader_train, optimizer, device, epoch, args.clip_max_norm) lr_scheduler.step() if args.output_dir: checkpoint_paths = [output_dir / 'checkpoint.pth'] # extra checkpoint before LR drop and every 100 epochs if (epoch + 1) % args.lr_drop == 0 or (epoch + 1) % 100 == 0: checkpoint_paths.append(output_dir / f'checkpoint{epoch:04}.pth') for checkpoint_path in checkpoint_paths: utils.save_on_master( { 'model': model_without_ddp.state_dict(), 'optimizer': optimizer.state_dict(), 'lr_scheduler': lr_scheduler.state_dict(), 'epoch': epoch, 'args': args, }, checkpoint_path) if (epoch + 1) % args.eval_freq == 0: test_stats, coco_evaluator = evaluate(model, criterion, postprocessors, data_loader_val, base_ds, device, args.output_dir) log_stats = { **{f'train_{k}': v for k, v in train_stats.items()}, **{f'test_{k}': v for k, v in test_stats.items()}, 'epoch': epoch, 'n_parameters': n_parameters } if args.output_dir and utils.is_main_process(): with (output_dir / "log.txt").open("a") as f: f.write(json.dumps(log_stats) + "\n") # for evaluation logs if coco_evaluator is not None: (output_dir / 'eval').mkdir(exist_ok=True) if "bbox" in coco_evaluator.coco_eval: filenames = ['latest.pth'] if epoch % 50 == 0: filenames.append(f'{epoch:03}.pth') for name in filenames: torch.save(coco_evaluator.coco_eval["bbox"].eval, output_dir / "eval" / name) total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=int(total_time))) logger.info('Training time {}'.format(total_time_str))
from channel.mq_client import MsgQueue from util.constants import Constants from util.db import DB, Collection from util.dump_stack import dumpstacks from util.logger import setup_logger from util.plot import HistoryPlot as Plot from datetime import datetime import json import logging import time setup_logger('backtest.log') _logger = logging.getLogger(__name__) class SingleBackTest(object): def __init__(self, stock_code): self.__stock_code = stock_code self.__mq = MsgQueue() def start(self): self.__mq.start() def stop(self): self.__mq.stop() def test(self, start, end): stock_info = {'stock': self.__stock_code, 'start': start, 'end': end} self.__mq.send(json.dumps(stock_info)) time.sleep(1) msg = self.__mq.recv()
from util.db import DB, Collection from util.config import Config from util.constants import Constants from util.file_db import FileDB from util.logger import setup_logger from util.dump_stack import dumpstacks from util.util import Util from collections import OrderedDict from datetime import datetime, timedelta import json import logging import pymongo import time setup_logger('strategy_engine.log') _logger = logging.getLogger(__name__) class StrategyEngine(object): DEFAULT_STRATEGY = 'macd_strategy' DEFAULT_BAR_PERIOD = 30 def __init__(self): self.__mq_server = None self.__data_db = DB(Constants.HIST_DATA_DB_NAME) self.__config = Config() self.__tick_db = FileDB( self.__config.get_config('persistent', 'hist_tick_dir')) self.__trading_strategy = None self.__tick_collector = None
app = Flask(__name__) gauge_db = GaugeDB() # Needed for single main.py file THREADED_RUN = True # Make 80 for AWS EC2, default is 5000 PORT = 80 # Make 0.0.0.0 to IP redirect, default is 127.0.0.1 HOST = '0.0.0.0' DEBUG_SERVER = False # Logger logfix = setup_logger() currencies = ["GBP/USD","EUR/USD","USD/CHF","USD/JPY","USD/CAD","USD/SGD","AUD/USD"] try: g_data = gauge_db.get_latest_gauge() currjson = g_data.get("currencies") gauge = g_data.get("GAU") except: currjson = { "GBP":0.0, "EUR":0.0, "CHF":0.0, "JPY":0.0, "CAD":0.0, "SGD":0.0,
from util.db import DB, Collection from util.config import Config from util.constants import Constants from util.file_db import FileDB from util.logger import setup_logger from util.dump_stack import dumpstacks from util.util import Util from collections import OrderedDict from datetime import datetime, timedelta import json import logging import pymongo import time setup_logger('strategy_engine.log') _logger = logging.getLogger(__name__) class StrategyEngine(object): DEFAULT_STRATEGY = 'macd_strategy' DEFAULT_BAR_PERIOD = 30 def __init__(self): self.__mq_server = None self.__data_db = DB(Constants.HIST_DATA_DB_NAME) self.__config = Config() self.__tick_db = FileDB(self.__config.get_config('persistent', 'hist_tick_dir')) self.__trading_strategy = None self.__tick_collector = None def start(self): Util.set_token()
from collector.hist_data_collector import HistDataCollector from collector.stock_basic_collector import StockBasicCollector from util.constants import Constants from util.db import DB, Collection from util.logger import setup_logger from apscheduler.schedulers.blocking import BlockingScheduler import time import logging setup_logger('period_gather.log') _logger = logging.getLogger(__name__) class PeriodGather(object): def __init__(self, db): self.__db = db self.__scheduler = BlockingScheduler() self.__scheduler.add_job(self.gather, 'cron', day_of_week='mon-fri', hour=16, minute=30) def start(self): self.__scheduler.start() def gather(self): _logger.info('period gather stock basic and history data, begin.....')
def main(): parser = argparse.ArgumentParser(description='Chainer example: MNIST') parser.add_argument('--batchsize', '-b', type=int, default=32, help='Number of images in each mini-batch') parser.add_argument('--epoch', '-e', type=int, default=30, help='Number of sweeps over the dataset to train') parser.add_argument('--centerloss', '-c', action='store_true', default=False, help='Use center loss') parser.add_argument('--alpha_ratio', '-a', type=float, default=0.5, help='alpha ratio') parser.add_argument('--lambda_ratio', '-l', type=float, default=0.1, help='lambda ratio') parser.add_argument('--frequency', '-f', type=int, default=-1, help='Frequency of taking a snapshot') parser.add_argument('--gpu', '-g', type=int, default=-1, help='GPU ID (negative value indicates CPU)') parser.add_argument('--out', '-o', default='result', help='Directory to output the result') parser.add_argument('--resume', '-r', default='', help='Resume the training from snapshot') args = parser.parse_args() logger = setup_logger(__name__) logger.info("GPU: {}".format(args.gpu)) logger.info("# Minibatch-size: {}".format(args.batchsize)) logger.info("# epoch: {}".format(args.epoch)) logger.info("Calculate center loss: {}".format(args.centerloss)) if args.centerloss: logger.info('# alpha: {}'.format(args.alpha_ratio)) logger.info('# lambda: {}'.format(args.lambda_ratio)) NUM_CLASSES = 10 model = LeNets( out_dim=NUM_CLASSES, alpha_ratio=args.alpha_ratio, lambda_ratio=args.lambda_ratio, is_center_loss=args.centerloss, ) if args.gpu >= 0: # Make a specified GPU current chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() # Copy the model to the GPU # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) # Load the MNIST dataset train, test = chainer.datasets.get_mnist(ndim=3) train_iter = chainer.iterators.MultiprocessIterator(train, args.batchsize, n_processes=4) test_iter = chainer.iterators.MultiprocessIterator(test, args.batchsize, n_processes=4, repeat=False, shuffle=False) # Set up a trainer updater = training.StandardUpdater(train_iter, optimizer, device=args.gpu) trainer = training.Trainer(updater, (args.epoch, 'epoch'), out=args.out) # Evaluate the model with the test dataset for each epoch trainer.extend(extensions.Evaluator(test_iter, model, device=args.gpu)) # Dump a computational graph from 'loss' variable at the first iteration # The "main" refers to the target link of the "main" optimizer. trainer.extend(extensions.dump_graph('main/loss')) # Take a snapshot for each specified epoch frequency = args.epoch if args.frequency == -1 else max(1, args.frequency) trainer.extend(extensions.snapshot(), trigger=(frequency, 'epoch')) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.LogReport()) # Save two plot images to the result dir if extensions.PlotReport.available(): trainer.extend( extensions.PlotReport(['main/loss', 'validation/main/loss'], 'epoch', file_name='loss.png')) trainer.extend( extensions.PlotReport( ['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy.png')) # Print selected entries of the log to stdout # Here "main" refers to the target link of the "main" optimizer again, and # "validation" refers to the default name of the Evaluator extension. # Entries other than 'epoch' are reported by the Classifier link, called by # either the updater or the evaluator. trainer.extend( extensions.PrintReport([ 'epoch', 'iteration', 'main/loss', 'validation/main/loss', 'main/accuracy', 'validation/main/accuracy', 'elapsed_time' ])) # Visualize Deep Features trainer.extend(VisualizeDeepFeature(train[:10000], NUM_CLASSES, args.centerloss), trigger=(1, 'epoch')) # Print a progress bar to stdout trainer.extend(extensions.ProgressBar()) if args.resume: # Resume from a snapshot chainer.serializers.load_npz(args.resume, trainer) # Run the training trainer.run()
from channel.mq_client import MsgQueue from util.constants import Constants from util.db import DB, Collection from util.dump_stack import dumpstacks from util.logger import setup_logger from util.plot import HistoryPlot as Plot from datetime import datetime import json import logging import time setup_logger('backtest.log') _logger = logging.getLogger(__name__) class SingleBackTest(object): def __init__(self, stock_code): self.__stock_code = stock_code self.__mq = MsgQueue() def start(self): self.__mq.start() def stop(self): self.__mq.stop() def test(self, start, end): stock_info = {'stock': self.__stock_code, 'start': start, 'end': end} self.__mq.send(json.dumps(stock_info)) time.sleep(1)
from collector.hist_data_collector import HistDataCollector from collector.stock_basic_collector import StockBasicCollector from util.constants import Constants from util.db import DB, Collection from util.logger import setup_logger from apscheduler.schedulers.blocking import BlockingScheduler import time import logging setup_logger('period_gather.log') _logger = logging.getLogger(__name__) class PeriodGather(object): def __init__(self, db): self.__db = db self.__scheduler = BlockingScheduler() self.__scheduler.add_job(self.gather, 'cron', day_of_week='mon-fri', hour=16, minute=30) def start(self): self.__scheduler.start() def gather(self): _logger.info('period gather stock basic and history data, begin.....') try: StockBasicCollector(self.__db).collect() stock_list = self.__get_stock_list() for stock in stock_list: HistDataCollector(stock, self.__db).collect()
import os from bunq.sdk import context import util.logger as logger from api.client import Client from bot.bot import TelegramBot from func.interface import BudgetApiInterface _API_KEY = os.environ['BUNQ_BOT_API_KEY'] _BOT_TOKEN = os.environ['BUNQ_BOT_TOKEN'] _DEVICE_DESCRIPTION = 'Better Bunq Bot' _LOG_FILENAME = 'better-bunq-bot.log' # Change this one, once you're ready to leave the SandBox! _ENVIRONMENT = context.ApiEnvironmentType.PRODUCTION if __name__ == "__main__": logger.setup_logger(_LOG_FILENAME) Client.setup_api_context(_ENVIRONMENT, _API_KEY, _DEVICE_DESCRIPTION) budget_interface = BudgetApiInterface() bot = TelegramBot(_BOT_TOKEN, budget_interface)