def test_build_colorized_logger_with_splunk_user_and_password(self): """test_build_colorized_logger""" log = build_colorized_logger(name='build_colorized_logger_with_splunk', splunk_user='******', splunk_password='******', splunk_sleep_interval=0) self.assertIsNotNone(log)
def run_main(): """run_main""" log = build_colorized_logger( name='helloworld', splunk_user='******', splunk_password='******', # handler_name='simple', # handler_name='not-real', # splunk_address='localhost:8088', # splunk_token='55df5127-cb0e-4182-932e-c71c454699b8', splunk_debug=False) max_recs = 1 msg_sent = 0 not_done = True while not_done: log.debug('testing DEBUG message_id={}'.format(str(uuid.uuid4()))) log.info('testing INFO message_id={}'.format(str(uuid.uuid4()))) log.error('testing ERROR message_id={}'.format(str(uuid.uuid4()))) log.critical('testing CRITICAL message_id={}'.format(str( uuid.uuid4()))) log.warning('testing WARNING message_id={}'.format(str(uuid.uuid4()))) try: raise Exception('Throw for testing exceptions') except Exception as e: log.error(('Testing EXCEPTION with ex={} message_id={}').format( e, str(uuid.uuid4()))) # end of try/ex msg_sent += 6 if msg_sent >= max_recs: not_done = False # end of while """ The threaded and multiprocessing Splunk Publishers require an exit sleep to prevent message loss when the parent process exits. The other handler(s) do not. """ wait_for_exit.wait_for_exit(log)
import glob import copy import random import pandas as pd from spylunking.log.setup_logging import build_colorized_logger from antinex_utils.consts import VALID from antinex_utils.consts import INVALID from antinex_utils.utils import ev from antinex_utils.utils import ppj from antinex_utils.utils import rnow log = build_colorized_logger(name='prepare_dataset_tools') def find_all_headers(use_log_id=None, pipeline_files=[], label_rules=None): """find_all_headers :param use_log_id: label for debugging in logs :param pipeline_files: list of files to prep :param label_rules: dict of rules to apply """ log_id = "" if use_log_id: log_id = use_log_id log.info(("{} find_all_headers - START").format(log_id)) headers = ["src_file"] headers_dict = {"src_file": None}
export SHARED_LOG_CFG=/opt/sa/analysis_engine/log/debug-logging.json """ import redis import pandas as pd import analysis_engine.build_result as build_result import analysis_engine.get_data_from_redis_key as redis_get from spylunking.log.setup_logging import build_colorized_logger from analysis_engine.consts import SUCCESS from analysis_engine.consts import NOT_RUN from analysis_engine.consts import ERR from analysis_engine.consts import ev from analysis_engine.consts import ppj log = build_colorized_logger(name=__name__) def build_df_from_redis(label=None, client=None, address=None, host=None, port=None, password=None, db=None, key=None, expire=None, serializer='json', encoding='utf-8', orient='records'): """build_df_from_redis
.. note:: This command pushes data to the webapp in the other terminal listening on port ``3434`` :: vprof -c cm ./analysis_engine/perf/profile_algo_runner.py """ import datetime import vprof.runner as perf_runner import analysis_engine.consts as ae_consts import analysis_engine.algo_runner as algo_runner import spylunking.log.setup_logging as log_utils log = log_utils.build_colorized_logger(name='profile-algo') def start(): """start""" back_a_few_days = (datetime.datetime.now() - datetime.timedelta(days=3)) start_date = back_a_few_days.strftime(ae_consts.COMMON_DATE_FORMAT) ticker = 'SPY' s3_bucket = 'perftests' s3_key = (f'{ticker}_{start_date}') algo_config = (f'./cfg/default_algo.json') history_loc = (f's3://{s3_bucket}/{s3_key}')
:: backtest_with_runner.py -l -t TICKER -b S3_BUCKET -k S3_KEY -c ALGO_CONFIG Debug by adding ``-d`` as an argument """ import sys import argparse import analysis_engine.utils as ae_utils import analysis_engine.algo_runner as algo_runner import analysis_engine.plot_trading_history as plot import spylunking.log.setup_logging as log_utils log = log_utils.build_colorized_logger(name='algo-runner') def backtest_with_runner(): """backtest_with_runner build and publish a trading history from an algorithm config. :: backtest_with_runner.py -t TICKER -c ALGO_CONFIG -s START_DATE -k S3_KEY -b S3_BUCKET -l """ parser = argparse.ArgumentParser( description=('backtest an algorithm and publish '
import analysis_engine.iex.consts as iex_consts import analysis_engine.api_requests as api_requests import analysis_engine.work_tasks.get_new_pricing_data as task_pricing import analysis_engine.work_tasks.task_screener_analysis as screener_utils import analysis_engine.utils as ae_utils import spylunking.log.setup_logging as log_utils # Disable celery log hijacking # https://github.com/celery/celery/issues/2509 @celery.signals.setup_logging.connect def setup_celery_logging(**kwargs): pass log = log_utils.build_colorized_logger( name='fetch', log_config_path=ae_consts.LOG_CONFIG_PATH) def start_screener_analysis(req): """start_screener_analysis Start screener-driven analysis with a simple workflow: 1) Convert FinViz screeners into a list of tickers and a ``pandas.DataFrames`` from each ticker's html row 2) Build unique list of tickers 3) Pull datasets for each ticker 4) Run sale-side processing - coming soon 5) Run buy-side processing - coming soon 6) Issue alerts to slack - coming soon
a file in s3 from running the backtester with the save to file option enabled: :: run_backtest_and_plot_history.py -t SPY -f <SAVE_HISTORY_TO_THIS_FILE> """ import argparse import pandas as pd import analysis_engine.consts as consts import analysis_engine.plot_trading_history as plot_trading_history import analysis_engine.load_history_dataset as load_history import spylunking.log.setup_logging as log_utils log = log_utils.build_colorized_logger(name='view-history-in-s3') def plot_history_from_s3(): """plot_history_from_s3 Run a derived algorithm with an algorithm config dictionary :param config_dict: algorithm config dictionary """ log.debug('start - plot') parser = argparse.ArgumentParser( description=('plot a local algorithm trading history file')) parser.add_argument('-b',
import json import pandas as pd from celery import Celery from spylunking.log.setup_logging import build_colorized_logger from antinex_utils.consts import SUCCESS from antinex_utils.consts import ERROR from antinex_utils.consts import FAILED log = build_colorized_logger( name='send_results') def send_results_to_broker( loc, final_results): """send_results_to_broker :param loc: api-generated dictionary for where to send the results :param final_results: prediction results from the worker """ log.info(( 'sending back={}').format( loc)) status = ERROR org_model = None org_rounded = None org_train_scaler = None org_test_scaler = None
import os import json import numpy as np import pandas as pd from spylunking.log.setup_logging import build_colorized_logger from antinex_utils.utils import ev from antinex_utils.consts import VALID from antinex_utils.consts import INVALID from antinex_utils.consts import ERROR from sklearn.model_selection import train_test_split log = build_colorized_logger(name='build_training_request') def build_training_request( csv_file=ev("CSV_FILE", "/tmp/cleaned_attack_scans.csv"), meta_file=ev("CSV_META_FILE", "/tmp/cleaned_metadata.json"), predict_feature=ev("PREDICT_FEATURE", "label_value"), ignore_features=[ "label_name", "ip_src", # need to make this an int "ip_dst", # need to make this an int "eth_src", # need to make this an int "eth_dst" # need to make this an int ], seed=None, test_size=float(ev("TEST_SIZE", "0.20")), preproc_rules=None): """build_training_request :param csv_file: csv file built with prepare_dataset.py
from analysis_engine.consts import REDIS_PASSWORD from analysis_engine.consts import REDIS_DB from analysis_engine.consts import REDIS_EXPIRE from analysis_engine.consts import get_status from analysis_engine.consts import ppj from analysis_engine.consts import is_celery_disabled # Disable celery log hijacking # https://github.com/celery/celery/issues/2509 @signals.setup_logging.connect def setup_celery_logging(**kwargs): pass log = build_colorized_logger(name='pub-tic-agg-s3-to-redis', log_config_path=LOG_CONFIG_PATH) def publish_ticker_aggregate_from_s3(): """publish_ticker_aggregate_from_s3 Download all ticker data from S3 and publish it's contents to Redis and back to S3 """ log.info('start - publish_ticker_aggregate_from_s3') parser = argparse.ArgumentParser( description=('Download and aggregated all ticker data, ' 'and store it in S3 and Redis. '))
#!/usr/bin/env python import celery import analysis_engine.work_tasks.get_celery_app as get_celery_app import analysis_engine.consts as consts import spylunking.log.setup_logging as log_utils # Disable celery log hijacking # https://github.com/celery/celery/issues/2509 @celery.signals.setup_logging.connect def setup_celery_logging(**kwargs): pass log = log_utils.build_colorized_logger(name=consts.APP_NAME, log_config_path=consts.LOG_CONFIG_PATH) log.info('start - {}'.format(consts.APP_NAME)) log.info('broker={} backend={} ' 'config={} include_tasks={}'.format( consts.WORKER_BROKER_URL, consts.WORKER_BACKEND_URL, consts.WORKER_CELERY_CONFIG_MODULE, consts.WORKER_TASKS)) # Get the Celery app from the project's get_celery_app module app = get_celery_app.get_celery_app( name=consts.APP_NAME, path_to_config_module=consts.WORKER_CELERY_CONFIG_MODULE, auth_url=consts.WORKER_BROKER_URL, backend_url=consts.WORKER_BACKEND_URL, include_tasks=consts.INCLUDE_TASKS)
def test_build_colorized_logger_without_splunk(self): """test_build_colorized_logger""" log = build_colorized_logger( name='build_colorized_logger_without_splunk', splunk_sleep_interval=0) self.assertIsNotNone(log)
import datetime import json import pandas as pd import antinex_utils.make_predictions from spylunking.log.setup_logging import build_colorized_logger from antinex_utils.utils import ppj from antinex_utils.consts import SUCCESS from antinex_core.send_results_to_broker import send_results_to_broker log = build_colorized_logger(name='processor') class AntiNexProcessor: """ AntiNexProcessor handles messages found in the subscribed queues. Conceptually ``Wokers use a Processor`` to handle messages. This one is responsible for processing ``Prediction`` and ``Training`` messages. It also manages a dictionary (``self.models``) of pre-trained deep neural networks for reused by a ``label`` name inside the consumed JSON dictionary message. """ def __init__(self, name="prc", max_msgs=100, max_models=100): """__init__ :param name: log label :param max_msgs: num msgs to save for replay debugging (FIFO) :param max_models: num pre-trained models to keep in memory (FIFO) """
a locally saved file from running the backtester with the save to file option enabled: :: run_backtest_and_plot_history.py -t SPY -f <SAVE_HISTORY_TO_THIS_FILE> """ import os import argparse import pandas as pd import analysis_engine.consts as ae_consts import analysis_engine.plot_trading_history as plot_trading_history import spylunking.log.setup_logging as log_utils log = log_utils.build_colorized_logger( name='plot-history', log_config_path=ae_consts.LOG_CONFIG_PATH) def plot_local_history_file(): """plot_local_history_file Run a derived algorithm with an algorithm config dictionary :param config_dict: algorithm config dictionary """ log.debug('start - plot') parser = argparse.ArgumentParser( description=('plot a local algorithm trading history file')) parser.add_argument('-f',
import pandas as pd import matplotlib.pyplot as plt import seaborn as sns from keras.models import Sequential from keras.layers import Dense from keras.wrappers.scikit_learn import KerasClassifier from sklearn.model_selection import cross_val_score from sklearn.model_selection import StratifiedKFold from sklearn.preprocessing import StandardScaler from sklearn.preprocessing import MinMaxScaler from sklearn.pipeline import Pipeline from sklearn.metrics import confusion_matrix from sklearn.model_selection import train_test_split from spylunking.log.setup_logging import build_colorized_logger log = build_colorized_logger(name='standalone_scaler_django') def build_model(num_features, loss, optimizer, metrics): """build_model Build the Keras Deep Neural Network Model :param num_features: number of features :param loss: loss function to apply :param optimizer: optimizer to use :param metrics: list of metrics """ log.info("building model") model = Sequential() model.add(
from celery.task import task from spylunking.log.setup_logging import build_colorized_logger from celery_loaders.work_tasks.custom_task import CustomTask log = build_colorized_logger(name='always_fails_tasks') @task(bind=True, base=CustomTask, queue="always_fails") def always_fails(self, work_dict): """always_fails :param work_dict: dictionary for key/values """ label = "always_fails" log.info(("task - {} - start " "work_dict={}").format(label, work_dict)) raise Exception(work_dict.get("test_failure", "simulating a failure")) log.info(("task - {} - done").format(label)) return True # end of always_fails
#!/usr/bin/env python import os import sys import datetime import argparse import json import pandas as pd from spylunking.log.setup_logging import build_colorized_logger from celery_connectors.publisher import Publisher log = build_colorized_logger(name='publish_regression_predict') def publish_regression_prediction_to_broker(): """publish_regression_prediction_to_broker Publish a Regression Prediction message to the Celery Worker's broker queue. This message is a JSON Dictionary. Default Broker: ``redis://localhost:6379/6`` Default Exchange: ``webapp.predict.requests`` Default Routing Key: ``webapp.predict.requests`` Default Queue: ``webapp.predict.requests`` """ parser = argparse.ArgumentParser(description=("Launch a Predict Dataset " "Request into the AntiNex " "core")) parser.add_argument("-f",
import pandas as pd from sklearn.preprocessing import MinMaxScaler from spylunking.log.setup_logging import build_colorized_logger from antinex_utils.consts import SUCCESS from antinex_utils.consts import ERR from antinex_utils.consts import NOTRUN log = build_colorized_logger(name='build_scaler_dataset') def build_scaler_dataset_from_records(record_list, label="build-scaled-dataset", min_feature=-1, max_feature=1, cast_to_type="float32"): """build_scaler_dataset_from_records :param record_list: list of json records to scale between min/max :param label: log label for tracking :param min_feature: min feature range for scale normalization :param max_feature: max feature range for scale normalization :param cast_to_type: cast all of the dataframe to this datatype """ status = NOTRUN last_step = "not-run" df = None scaler = None dataset = None res = {
import celery from spylunking.log.setup_logging import build_colorized_logger log = build_colorized_logger( name='custom-task') class CustomTask(celery.Task): """CustomTask""" log_label = "custom_task" def on_success(self, retval, task_id, args, kwargs): """on_success http://docs.celeryproject.org/en/latest/reference/celery.app.task.html :param retval: return value :param task_id: celery task id :param args: arguments passed into task :param kwargs: keyword arguments passed into task """ log.info(("{} SUCCESS - retval={} task_id={} " "args={} kwargs={}") .format( self.log_label, retval, task_id, args,
#!/usr/bin/env python import os import uuid from spylunking.log.setup_logging import build_colorized_logger from celery_connectors.redis.redis_json_application import RedisJSONApplication port = 6379 host = os.getenv("ENV_REDIS_HOST", "localhost").strip().lstrip() db = int(os.getenv("ENV_REDIS_DB_ID", 0)) # if set to empty string use password=None redis_pw = os.getenv("ENV_REDIS_PASSWORD", "") queue_name = os.getenv("Q_1", "reporting.accounts").strip().lstrip() name = "redis-producer" log = build_colorized_logger( name='redis-publish') log.info("START - {} - Sending messages to redis={}:{}/{} queue={}".format(name, host, port, db, queue_name)) if str(redis_pw) == "": redis_pw = None app = RedisJSONApplication(name, redis_address=host, redis_port=port, redis_queue=queue_name, logger=log) app.connect() max_msgs = 1000 msgs = []
def __init__(self, config_dict, path_to_module=None, name=None, verbose=False): """__init__ Base class for building your own indicators to work within an ``IndicatorProcessor``. Please derive the ``self.process()`` method as needed .. tip:: any keys passed in with ``config_dict`` will become class member variables that can be accessed and used as normal member variables within the derived Indicator class :param config_dict: dictionary for this indicator :param name: name of the indicator :param path_to_module: work in progress - this will allow loading indicators from outside the repo like the derived algorithm classes :param verbose: optional - bool for toggling more logs """ self.name = name self.log = log_utils.build_colorized_logger(name=name) self.config = config_dict self.path_to_module = path_to_module self.verbose = verbose if not self.config: raise Exception('please provide a config_dict for loading ' 'the buy and sell rules for this indicator') if not self.verbose: self.verbose = self.config.get('verbose', False) if not self.name: self.name = 'ind_{}'.format(str(uuid.uuid4()).replace('-', '')) self.starter_dict = None self.previous_df = self.config.get('previous_df', None) self.name_of_df = self.config.get('uses_data', None) self.uses_data = self.name_of_df self.report = self.config.get('report', {}) self.ind_id = self.report.get('id', self.name) self.metrics = self.report.get('metrics', {}) self.ind_type = self.metrics.get('type', ae_consts.INDICATOR_TYPE_UNKNOWN) self.ind_category = self.metrics.get( 'category', ae_consts.INDICATOR_CATEGORY_UNKNOWN) self.ind_uses_data = self.metrics.get( 'ind_uses_data', ae_consts.INDICATOR_USES_DATA_ANY) self.dataset_df_str = self.config.get('dataset_df', None) self.report_key_prefix = self.report.get('report_key_prefix', self.name) # this should be mostly numeric values # to allow converting to an AI-ready dataset # once the algorithm finishes self.report_dict = { 'type': self.ind_type, 'category': self.ind_category, 'uses_data': self.ind_uses_data } self.report_ignore_keys = self.config.get( 'report_ignore_keys', ae_consts.INDICATOR_IGNORED_CONIGURABLE_KEYS) self.use_df = pd.DataFrame(ae_consts.EMPTY_DF_LIST) self.configurables = self.config self.ind_confs = [] self.convert_config_keys_to_members()
#!/usr/bin/env python import os import json from celery import Celery from django.conf import settings from spylunking.log.setup_logging import build_colorized_logger from antinex_utils.utils import ppj name = 'send-worker-publish-to-core' log = build_colorized_logger( name=name) os.environ.setdefault( "DJANGO_SETTINGS_MODULE", "drf_network_pipeline.settings") log.info("creating celery app") app = Celery("test-app") app.config_from_object( "django.conf:settings", namespace="CELERY") app.autodiscover_tasks( lambda: settings.INSTALLED_APPS) datafile = "./drf_network_pipeline/tests/pubsub/publish-to-core.json" data = {} with open(datafile, "r") as f:
import json import pandas as pd from spylunking.log.setup_logging import build_colorized_logger from antinex_utils.consts import SUCCESS from antinex_utils.consts import ERR from antinex_utils.consts import NOTRUN log = build_colorized_logger(name='merge_inverse_data_into_original') def merge_inverse_data_into_original(req, sort_on_index=None, ordered_columns=None): """merge_inverse_data_into_original :param req: managed dictionary :param sort_on_index: sort the dataframe on this column name :param ordered_columns: column list to rename the inverse transform """ label = req.get("label", "") last_step = "not-run" status = NOTRUN org_df = None predict_df = None merge_df = None res = { "status": status, "err": last_step, "sorted_org_df": org_df, "predict_df": predict_df, "merge_df": merge_df
examine IEX Cloud company data, calls = examine Tradier calls data, puts = examine Tradier puts data, and comma delimited is supported as well -s START_DATE start date format YYYY-MM-DD (default is 2019-01-01) """ import datetime import argparse import analysis_engine.consts as ae_consts import analysis_engine.utils as ae_utils import analysis_engine.extract as ae_extract import spylunking.log.setup_logging as log_utils log = log_utils.build_colorized_logger(name='inspect-redis-data', handler_name='no_date_colors') def inspect_datasets(ticker=None, start_date=None, datasets=None): """inspect_datasets Loop over all cached data in redis by going sequentially per date and examine the latest ``date`` value in the cache to check if it matches the redis key's date. For IEX Cloud minute data errors, running this function will print out commands to fix any issues (if possible): :: fetch -t TICKER -g iex_min -F DATE_TO_FIX
from celery import Celery from celery import signals from spylunking.log.setup_logging import build_colorized_logger from antinex_utils.utils import ev from celery_connectors.subscriber import Subscriber from antinex_core.antinex_processor import AntiNexProcessor # Disable celery log hijacking # https://github.com/celery/celery/issues/2509 @signals.setup_logging.connect def setup_celery_logging(**kwargs): pass log = build_colorized_logger(name='worker') class AntiNexCore: """ AntiNex Celery Worker Core (core) This is a Celery Worker used to connect to a message broker (``BROKER_URL=redis://localhost:6379/6`` by default) and monitor messages to consume off the following queues: ``TRAIN_QUEUE`` - ``webapp.train.requests`` ``PREDICT_QUEUE`` - ``webapp.predict.requests`` The core trains and manages pre-trained deep neural networks (dnn) by
#!/usr/bin/env python import os import sys import json import argparse import pandas as pd from spylunking.log.setup_logging import build_colorized_logger from antinex_utils.utils import ppj log = build_colorized_logger(name='convert_bottom_rows_to_json') def convert_bottom_rows_to_json(): """convert_bottom_rows_to_json Convert the last few rows in a dataset to JSON """ parser = argparse.ArgumentParser(description=("Convert the last few " "rows in a dataset to JSON")) parser.add_argument( "-f", help=( "dataset to use default /opt/antinex/antinex-datasets/v1/webapps/" "django/training-ready/v1_django_cleaned.csv"), required=False, dest="dataset") parser.add_argument("-b", help=("rows from bottom of dataset"), required=False,
import tensorflow as tf import analysis_engine.consts as ae_consts import analysis_engine.load_history_dataset as load_history import analysis_engine.ai.build_regression_dnn as build_dnn import analysis_engine.ai.build_datasets_using_scalers as build_scaler_datasets import analysis_engine.ai.build_scaler_dataset_from_df as build_scaler_df import analysis_engine.ai.plot_dnn_fit_history as plot_fit_history import analysis_engine.plot_trading_history as plot_trading_history import spylunking.log.setup_logging as log_utils from copy import deepcopy # ensure reproducible results # machinelearningmastery.com/reproducible-results-neural-networks-keras/ np_random.seed(1) log = log_utils.build_colorized_logger(name='train-dnn-from-history') choices = ['close', 'high', 'low'] def train_and_predict_from_history_in_s3(): """train_and_predict_from_history_in_s3 Run a derived algorithm with an algorithm config dictionary :param config_dict: algorithm config dictionary """ log.debug('start - plot') parser = argparse.ArgumentParser(
""" Helper for creating a buy order """ import analysis_engine.consts as ae_consts import analysis_engine.utils as ae_utils import spylunking.log.setup_logging as log_utils log = log_utils.build_colorized_logger(name=__name__) def build_buy_order(ticker, num_owned, close, balance, commission, date, details, use_key, minute=None, shares=None, version=1, auto_fill=True, is_live_trading=False, backtest_shares_default=10, reason=None): """build_buy_order Create an algorithm buy order as a dictionary .. note:: setting the ``minute`` is required to build
from antinex_utils.build_training_request import build_training_request from antinex_utils.make_predictions import make_predictions from drf_network_pipeline.users.db_lookup_user import \ db_lookup_user from drf_network_pipeline.pipeline.create_ml_prepare_record import \ create_ml_prepare_record from drf_network_pipeline.pipeline.process_worker_results import \ process_worker_results from drf_network_pipeline.job_utils.build_task_response import \ build_task_response from kombu import Connection from kombu import Producer from kombu import Exchange from kombu import Queue log = build_colorized_logger(name='ml.tasks') # allow tasks to be sent straight to the worker @shared_task(name=("drf_network_pipeline.pipeline.tasks." "task_ml_prepare"), queue=("drf_network_pipeline.pipeline.tasks." "task_ml_prepare"), bind=True) def task_ml_prepare(self=None, req_node=None): """task_ml_prepare :param self: parent task object for bind=True :param req_node: job utils dictionary for passing a dictionary """