def run_consumer():
logger = logging.getLogger('consumer')
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setFormatter(
logging.Formatter('%(asctime)-15s %(levelname)-8s %(message)s'))
logger.addHandler(handler)
consumer = Consumer(conf)
consumer.subscribe(topics=config.resolve_config("CONSUMER_TOPICS"))
try:
while True:
msg = consumer.poll(timeout=1.0)
if msg is None:
continue
if msg.error():
raise KafkaException(msg.error())
else:
# Proper message
# sys.stderr.write('%% %s [%d] at offset %d with key %s:\n' %
# (msg.topic(), msg.partition(), msg.offset(),
# str(msg.key())))
print(msg.value())
except KeyboardInterrupt:
sys.stderr.write('%% Aborted by user\n')
finally:
# Close down consumer to commit final offsets.
consumer.close()
def produce(message, topic=None, count=1):
_topic = topic or config.resolve_config("CONSUMER_TOPICS")
try:
for i in range(count):
producer.produce(_topic,
message + ": " + str(i),
on_delivery=on_delivery)
except Exception as e:
sys.stderr.write("Exception " + e.__class__.__name__ + " :: " + e)
def create_influx():
"""
Contextmanager that will create and teardown a session.
"""
config = resolve_config()
influx = InfluxDB(host=config.INFLUXDB_HOST, database=config.INFLUXDB_DATABASE)
yield influx
influx.close()
import sys
import config
from confluent_kafka.cimpl import Producer
conf = {'bootstrap.servers': config.resolve_config("BROKER")}
def on_delivery(err, msg):
if err:
sys.stderr.write('%% Message failed delivery: %s\n' % err)
else:
sys.stdout.write('%% Message delivered to %s [%d] @ %d\n' %
(msg.topic(), msg.partition(), msg.offset()))
producer = Producer(**conf)
def produce(message, topic=None, count=1):
_topic = topic or config.resolve_config("CONSUMER_TOPICS")
try:
for i in range(count):
producer.produce(_topic,
message + ": " + str(i),
on_delivery=on_delivery)
except Exception as e:
sys.stderr.write("Exception " + e.__class__.__name__ + " :: " + e)
default=200)
parser.add_argument("-v",
"--verbose",
help="allow verbosity",
action="store_true")
args = parser.parse_args()
def create_history(pair: str, timeframe: str, limit: int, verbose: bool):
series: Series = get_candles(pair=pair, timeframe=timeframe, limit=limit)
print(
f"Creating {series.pair}{series.timeframe}. Size: {series.close.size}")
atoms = Atomics(series)
atoms.remake(verbose)
if args.pair and args.timeframe:
verbose = True if args.verbose else False
# Setup proper config
Config = resolve_config()
# Create app
app = create_app(Config)
# Generate history
with app.app_context():
create_history(pair=args.pair,
timeframe=args.timeframe,
limit=args.limit,
verbose=verbose)
exit(0)
import logging
import sys
import os
import config
from confluent_kafka.cimpl import Consumer, KafkaException
conf = {
"bootstrap.servers": config.resolve_config("BROKER"),
"group.id": config.resolve_config("GROUP_ID"),
"session.timeout.ms": 6000,
"auto.offset.reset": "earliest",
"sasl.mechanisms": "PLAIN",
"security.protocol": "SASL_SSL",
"sasl.username": os.environ.get("CONF_API_KEY"),
"sasl.password": os.environ.get("CONF_API_SECRET")
}
def run_consumer():
logger = logging.getLogger('consumer')
logger.setLevel(logging.DEBUG)
handler = logging.StreamHandler()
handler.setFormatter(
logging.Formatter('%(asctime)-15s %(levelname)-8s %(message)s'))
logger.addHandler(handler)
consumer = Consumer(conf)
consumer.subscribe(topics=config.resolve_config("CONSUMER_TOPICS"))
try: