def handle_content(self):
pubsub, channel = create_pubsub('memory'), "chan1"
for idx, record in enumerate(self.iterator):
pattern = self.regex.search(dict_to_str(record)).group(1)
if pattern not in self._patterns:
logger.debug("Adding thread for pattern %s", pattern)
json_output = os.path.join(self.key, f"data-{pattern}.json")
if self.gzipped:
json_output += ".gz"
uploading_thread = thread(self.bucket, pubsub, channel,
pattern, json_output, self.gzipped,
self.buffer_size)
self._patterns[pattern] = 1
uploading_thread.start()
pubsub.publish(channel, (pattern, record))
logger.info(
"Main thread has published all data, %d records, coming from the database",
idx + 1)
while any([
subscriber.messages.qsize()
for subscriber in pubsub.subscribers[channel]
]):
logger.debug(
f"Waiting as there are still messages that have not been read by subscribers..."
)
time.sleep(10)
logger.debug(
"Main thread finished, sending message to all subscribers that there is no more data"
)
pubsub.publish(channel, (None, None))
def test_amqp_from_url(self):
self.pubsub = None
from anypubsub import create_pubsub
self.pubsub = create_pubsub('amqp', host='amqp://')
from anypubsub.backends.amqp import AmqpSubscriber
subscriber = self.pubsub.subscribe('a_chan')
assert isinstance(subscriber, AmqpSubscriber)
def test_redis_from_url(self):
self.pubsub = None
from anypubsub import create_pubsub
self.pubsub = create_pubsub('redis', host='redis://localhost:6379/0')
from anypubsub.backends.redis import RedisSubscriber
subscriber = self.pubsub.subscribe('a_chan')
assert isinstance(subscriber, RedisSubscriber)
def test_create_pubsub(self):
from anypubsub import create_pubsub
from anypubsub.backends.memory import MemoryPubSub
pubsub = create_pubsub('memory')
assert isinstance(pubsub, MemoryPubSub)
import signal
import sys
from socketio.namespace import BaseNamespace
from socketio.mixins import BroadcastMixin
from formatter import process_log
from web_server import make_server
import random
import datetime
q = Queue()
from anypubsub import create_pubsub
pubsub = create_pubsub('memory')
EQUIPS = ['RE002', 'RE003', 'RE004']
def gettimestr():
t = datetime.datetime.now()
return t.strftime('%Y-%m-%d %H:%M:%S')
def producer():
while True:
line = sys.stdin.readline()
process_log(line, q.put)
def dummy_producer():
i = 0
from gevent.queue import Queue
import signal
import sys
from socketio.namespace import BaseNamespace
from socketio.mixins import BroadcastMixin
from formatter import process_log
from web_server import make_server
import random
import datetime
q = Queue()
from anypubsub import create_pubsub
pubsub = create_pubsub('memory')
EQUIPS = ['RE002', 'RE003', 'RE004']
def gettimestr():
t = datetime.datetime.now()
return t.strftime('%Y-%m-%d %H:%M:%S')
def producer():
while True:
line = sys.stdin.readline()
process_log(line, q.put)
def setUp(self):
from anypubsub import create_pubsub
self.pubsub = create_pubsub('memory')
def test_create_pubsub(self):
from anypubsub import create_pubsub
from anypubsub.backends.memory import MemoryPubSub
pubsub = create_pubsub('memory')
assert isinstance(pubsub, MemoryPubSub)
def setUp(self):
from anypubsub import create_pubsub
self.pubsub = create_pubsub('mongodb')
def setUp(self):
from anypubsub import create_pubsub
self.pubsub = create_pubsub('amqp')
def setUp(self):
from anypubsub import create_pubsub
self.pubsub = create_pubsub('redis')
