def fixunacked(): returncode = 200 queue = HotQueue("access", host="localhost", port=6379, db=0) outputdata = "" for q in queue.get_unackedqueues(): if is_unacked_old(q): unackedqueue = HotQueue(q.replace('hotqueue:',''), host="localhost", port=6379, db=0) message_envelope = Message() message_envelope = unackedqueue.get() originalqueue = HotQueue(message_envelope.get_originalqueue(), host="localhost", port=6379, db=0) originalqueue.put_again(message_envelope) unackedqueue.clear() outputdata = outputdata+str(q)+':'+str(is_unacked_old(q))+"\n" if outputdata == "": returncode = 204 return outputdata, returncode
class AnmadQueues: """Queues used by anmad.""" def __init__(self, prequeue, queue, info): self.prequeue = HotQueue(prequeue) self.queue = HotQueue(queue) self.info = HotQueue(info) self.update_job_lists() def update_job_lists(self): """Reset queue_message vars.""" self.prequeue_list = [] self.prequeue_list = read_queue(self.prequeue) self.queue_list = [] self.queue_list = read_queue(self.queue) self.info_list = [] trim_queue(self.info, 100) self.info_list = read_queue(self.info) self.info_list.reverse() def prequeue_job(self, job): """Adds an item to the pre-run queue.""" self.prequeue.put([job]) def queue_job(self, job): """Adds an item to the run queue.""" self.queue.put(job) def clear(self): """Clears all job queues.""" self.queue.clear() self.prequeue.clear() def clearinfo(self): """Clears info queue.""" self.info.clear()
def init_workers(num): global scout, worker, parser # q contains list of threads, scraped by the scout q = HotQueue(home) # qf contains list of tuples of len 3 # contains source code of scraped pages and metadata, for parser # (subforum name, link of post, post source) qf = HotQueue(home + "_sources") # boolean variable that describes whether the Parser is training # (only used for GenericParser) qt = Value('b', False) workers = [] scout = multiprocessing.Process(target=scout, args=(q,)) scout.start() logger.info("Starting scout process %s" % str(scout.pid)) workers.append(scout) sleep(15) for i in xrange(num): tmp = multiprocessing.Process(target=worker, args=(q,qf,qt,delay, delay_range)) tmp.start() logger.info("Starting worker process %s" % str(tmp.pid)) workers.append(tmp) for i in xrange(num-1): tmp = multiprocessing.Process(target=parser, args=(qf,qt)) tmp.start() logger.info("Starting parser process %s" % str(tmp.pid)) workers.append(tmp) parser(qf,qt, main=True) logger.info("Using main process as parser") for worker in workers: worker.join() logger.info("All done.") q.clear() qf.clear()
class HotQueueTestCase(unittest.TestCase): def setUp(self): """Create the queue instance before the test.""" self.queue = HotQueue('testqueue') def tearDown(self): """Clear the queue after the test.""" self.queue.clear() def test_arguments(self): """Test that HotQueue.__init__ accepts arguments correctly, and that the Redis key is correctly formed. """ kwargs = { 'name': "testqueue", 'serializer': DummySerializer, 'host': "localhost", 'port': 6379, 'db': 0} # Instantiate the HotQueue instance: self.queue = HotQueue(**kwargs) # Ensure that the properties of the instance are as expected: self.assertEqual(self.queue.name, kwargs['name']) self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name']) self.assertEqual(self.queue.serializer, kwargs['serializer']) self.assertEqual(self.queue._HotQueue__redis.host, kwargs['host']) self.assertEqual(self.queue._HotQueue__redis.host, kwargs['host']) self.assertEqual(self.queue._HotQueue__redis.port, kwargs['port']) self.assertEqual(self.queue._HotQueue__redis.db, kwargs['db']) # Instantiate a HotQueue instance with only the required args: self.queue = HotQueue(kwargs['name']) # Ensure that the properties of the instance are as expected: self.assertEqual(self.queue.name, kwargs['name']) self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name']) self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle or # pickle, depending on the # platform. def test_consume(self): """Test the consume generator method.""" nums = [1, 2, 3, 4, 5, 6, 7, 8] # Test blocking with timeout: self.queue.put(*nums) msgs = [] for msg in self.queue.consume(timeout=1): msgs.append(msg) self.assertEquals(msgs, nums) # Test non-blocking: self.queue.put(*nums) msgs = [] for msg in self.queue.consume(block=False): msgs.append(msg) self.assertEquals(msgs, nums) def test_cleared(self): """Test for correct behaviour if the Redis list does not exist.""" self.assertEquals(len(self.queue), 0) self.assertEquals(self.queue.get(), None) def test_get_order(self): """Test that messages are get in the same order they are put.""" alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno'] self.queue.put(alphabet[0], alphabet[1], alphabet[2]) self.queue.put(alphabet[3]) self.queue.put(alphabet[4]) msgs = [] msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) self.assertEquals(msgs, alphabet) def test_length(self): """Test that the length of a queue is returned correctly.""" self.queue.put('a message') self.queue.put('another message') self.assertEquals(len(self.queue), 2) def test_worker(self): """Test the worker decorator.""" colors = ['blue', 'green', 'red', 'pink', 'black'] # Test blocking with timeout: self.queue.put(*colors) msgs = [] @self.queue.worker(timeout=1) def appender(msg): msgs.append(msg) appender() self.assertEqual(msgs, colors) # Test non-blocking: self.queue.put(*colors) msgs = [] @self.queue.worker(block=False) def appender(msg): msgs.append(msg) appender() self.assertEqual(msgs, colors) def test_threaded(self): """Threaded test of put and consume methods.""" msgs = [] def put(): for num in range(3): self.queue.put('message %d' % num) sleep(0.1) def consume(): for msg in self.queue.consume(timeout=1): msgs.append(msg) putter = threading.Thread(target=put) consumer = threading.Thread(target=consume) putter.start() consumer.start() for thread in [putter, consumer]: thread.join() self.assertEqual(msgs, ["message 0", "message 1", "message 2"]) def test_custom_serializer(self): """Test the use of a custom serializer.""" self.queue.serializer = DummySerializer phrase = "my message" self.queue.put(phrase) msg = self.queue.get() self.assertEqual(msg, phrase)
class HotQueueTestCase(unittest.TestCase): def setUp(self): """Create the queue instance before the test.""" self.queue = HotQueue('testqueue') def tearDown(self): """Clear the queue after the test.""" self.queue.clear() def test_arguments(self): """Test that HotQueue.__init__ accepts arguments correctly, and that the Redis key is correctly formed. """ kwargs = { 'name': "testqueue", 'serializer': DummySerializer, 'host': "localhost", 'port': 6379, 'db': 0} # Instantiate the HotQueue instance: self.queue = HotQueue(**kwargs) # Ensure that the properties of the instance are as expected: self.assertEqual(self.queue.name, kwargs['name']) self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name']) self.assertEqual(self.queue.serializer, kwargs['serializer']) # Instantiate a HotQueue instance with only the required args: self.queue = HotQueue(kwargs['name']) # Ensure that the properties of the instance are as expected: self.assertEqual(self.queue.name, kwargs['name']) self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name']) self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle # or pickle, depending # on the platform. def test_consume(self): """Test the consume generator method.""" nums = [1, 2, 3, 4, 5, 6, 7, 8] # Test blocking with timeout: self.queue.put(*nums) msgs = [] for msg in self.queue.consume(timeout=1): msgs.append(msg) self.assertEqual(msgs, nums) # Test non-blocking: self.queue.put(*nums) msgs = [] for msg in self.queue.consume(block=False): msgs.append(msg) self.assertEqual(msgs, nums) def test_cleared(self): """Test for correct behaviour if the Redis list does not exist.""" self.assertEqual(len(self.queue), 0) self.assertEqual(self.queue.get(), None) def test_get_order(self): """Test that messages are get in the same order they are put.""" alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno'] self.queue.put(alphabet[0], alphabet[1], alphabet[2]) self.queue.put(alphabet[3]) self.queue.put(alphabet[4]) msgs = [] msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) self.assertEqual(msgs, alphabet) def test_length(self): """Test that the length of a queue is returned correctly.""" self.queue.put('a message') self.queue.put('another message') self.assertEqual(len(self.queue), 2) def test_worker(self): """Test the worker decorator.""" colors = ['blue', 'green', 'red', 'pink', 'black'] # Test blocking with timeout: self.queue.put(*colors) msgs = [] @self.queue.worker(timeout=1) def appender(msg): msgs.append(msg) appender() self.assertEqual(msgs, colors) # Test non-blocking: self.queue.put(*colors) msgs = [] @self.queue.worker(block=False) def appender(msg): msgs.append(msg) appender() self.assertEqual(msgs, colors) # Test decorating a class method: self.queue.put(*colors) msgs = [] class MyClass(object): @self.queue.worker(block=False) def appender(self, msg): msgs.append(msg) my_instance = MyClass() my_instance.appender() self.assertEqual(msgs, colors) def test_threaded(self): """Threaded test of put and consume methods.""" msgs = [] def put(): for num in range(3): self.queue.put('message %d' % num) sleep(0.1) def consume(): for msg in self.queue.consume(timeout=1): msgs.append(msg) putter = threading.Thread(target=put) consumer = threading.Thread(target=consume) putter.start() consumer.start() for thread in [putter, consumer]: thread.join() self.assertEqual(msgs, ["message 0", "message 1", "message 2"]) def test_custom_serializer(self): """Test the use of a custom serializer and None as serializer.""" msg = "my message" # Test using None: self.queue.serializer = None self.queue.put(msg) self.assertEqual(self.queue.get().decode(), msg) self.queue.put({"a": 1}) expected = "{u'a': 1}" if sys.version_info[0] == 2 else "{'a': 1}" self.assertEqual(self.queue.get().decode(), expected) # Should be a string # Test using DummySerializer: self.queue.serializer = DummySerializer self.queue.put(msg) self.assertEqual(self.queue.get().decode(), "foo")
def clear_queue(): q = HotQueue(home) qf = HotQueue(home + "_sources") q.clear() qf.clear() os.remove(pfile)
os.mkdir(hdir) pfile = hdir[2:] + ".p" try: with open(pfile, "r") as f: state = cPickle.load(f) if args.generic: generic = True P = GenericParser(name=home,save=True) except: # no pickle file; fresh start state = [0, 0] q = HotQueue(home) qf = HotQueue(home + "_sources") q.clear() qf.clear() if args.generic: generic = True P = GenericParser(name=home,save=False) if args.vbulletin: vbulletin = True P = vBulletinParser() init_logger() temp = urlparse.urljoin("http:////", home) archive_link = urlparse.urljoin(temp, "/archive/index.php") atexit.register(save_state)
class HotQueueTestCase(unittest.TestCase): def setUp(self): """Create the queue instance before the test.""" self.queue = HotQueue('testqueue') def tearDown(self): """Clear the queue after the test.""" self.queue.clear() def test_arguments(self): """Test that HotQueue.__init__ accepts arguments correctly, and that the Redis key is correctly formed. """ kwargs = { 'name': "testqueue", 'serializer': DummySerializer, 'host': "localhost", 'port': 6379, 'db': 0} # Instantiate the HotQueue instance: self.queue = HotQueue(**kwargs) # Ensure that the properties of the instance are as expected: self.assertEqual(self.queue.name, kwargs['name']) self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name']) self.assertEqual(self.queue.serializer, kwargs['serializer']) # Instantiate a HotQueue instance with only the required args: self.queue = HotQueue(kwargs['name']) # Ensure that the properties of the instance are as expected: self.assertEqual(self.queue.name, kwargs['name']) self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs['name']) self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle # or pickle, depending # on the platform. def test_consume(self): """Test the consume generator method.""" nums = [1, 2, 3, 4, 5, 6, 7, 8] # Test blocking with timeout: nums_added = self.queue.put(*nums) msgs = [] for msg in self.queue.consume(timeout=1): msgs.append(msg) self.assertEqual(msgs, nums_added) for msg in msgs: self.queue.ack(msg.get_reservationId()) # Test non-blocking: nums_added = self.queue.put(*nums) msgs = [] for msg in self.queue.consume(block=False): msgs.append(msg) self.assertEqual(msgs, nums_added) for msg in msgs: self.queue.ack(msg.get_reservationId()) def test_nack(self): """Test the consume generator method.""" nums_added = self.queue.put("1") msg = self.queue.get() #1 self.assertEqual(msg, nums_added[0]) self.queue.nack(msg.get_reservationId()) msg = self.queue.get() #2 self.assertEqual(msg, nums_added[0]) self.queue.nack(msg.get_reservationId()) msg = self.queue.get() #3 if msg: self.assertEqual(msg, nums_added[0]) self.queue.ack(msg.get_reservationId()) self.assertEqual(msg, nums_added[0]) self.assertEqual(len(self.queue), 0) self.assertEqual(msg.get_deliveryCount(),3) #3 def test_cleared(self): """Test for correct behaviour if the Redis list does not exist.""" self.assertEqual(len(self.queue), 0) self.assertEqual(self.queue.get(), None) def test_get_order(self): """Test that messages are get in the same order they are put.""" alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno'] msg_added = [] msg_added.extend(self.queue.put(alphabet[0], alphabet[1], alphabet[2])) msg_added.extend(self.queue.put(alphabet[3])) msg_added.extend(self.queue.put(alphabet[4])) msgs = [] msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) self.assertEqual(msgs, msg_added) for msg in msgs: self.queue.ack(msg.get_reservationId()) def test_length(self): """Test that the length of a queue is returned correctly.""" self.queue.put('a message') self.queue.put('another message') self.assertEqual(len(self.queue), 2) def test_worker(self): """Test the worker decorator.""" colors = ['blue', 'green', 'red', 'pink', 'black'] # Test blocking with timeout: msg_added = self.queue.put(*colors) msgs = [] @self.queue.worker(timeout=1) def appender(msg): msgs.append(msg) appender() self.assertEqual(msgs, msg_added) for msg in msgs: self.queue.ack(msg.get_reservationId()) # Test non-blocking: msg_added = self.queue.put(*colors) msgs = [] @self.queue.worker(block=False) def appender(msg): msgs.append(msg) appender() self.assertEqual(msgs, msg_added) for msg in msgs: self.queue.ack(msg.get_reservationId()) # Test decorating a class method: msg_added = self.queue.put(*colors) msgs = [] class MyClass(object): @self.queue.worker(block=False) def appender(self, msg): msgs.append(msg) my_instance = MyClass() my_instance.appender() self.assertEqual(msgs, msg_added) for msg in msgs: self.queue.ack(msg.get_reservationId()) def test_threaded(self): """Threaded test of put and consume methods.""" msgs = [] msg_added = [] def put(): for num in range(3): msg_added.extend(self.queue.put('message %d' % num)) sleep(0.1) def consume(): for msg in self.queue.consume(timeout=1): msgs.append(msg) putter = threading.Thread(target=put) consumer = threading.Thread(target=consume) putter.start() consumer.start() for thread in [putter, consumer]: thread.join() self.assertEqual(msgs, msg_added) for msg in msgs: self.queue.ack(msg.get_reservationId())
import RPi.GPIO as GPIO import json import time HVAC = "ON" HVAC_STANDBY = 5 HVAC_RUN = 6 GPIO.cleanup() GPIO.setmode(GPIO.BCM) GPIO.setup(HVAC_STANDBY, GPIO.OUT) GPIO.setup(HVAC_RUN, GPIO.OUT) queue = HotQueue("sensordata", host="localhost", port=6379, db=0) queue.clear() def on_connect(mosq, obj, rc): mqttc.subscribe([("HVAC_STATUS", 0)]) print("rc: " + str(rc)) def on_message(mosq, obj, msg): global HVAC print(msg.topic + " " + str(msg.qos) + " " + str(msg.payload)) if msg.topic == 'HVAC_STATUS': HVAC = str(msg.payload)
import RPi.GPIO as GPIO import json import time HVAC = "ON" HVAC_STANDBY = 5 HVAC_RUN = 6 GPIO.cleanup() GPIO.setmode(GPIO.BCM) GPIO.setup(HVAC_STANDBY,GPIO.OUT) GPIO.setup(HVAC_RUN,GPIO.OUT) queue = HotQueue("sensordata", host="localhost", port=6379, db=0) queue.clear() def on_connect(mosq, obj, rc): mqttc.subscribe([("HVAC_STATUS", 0)]) print("rc: " + str(rc)) def on_message(mosq, obj, msg): global HVAC print(msg.topic + " " + str(msg.qos) + " " + str(msg.payload)) if msg.topic == 'HVAC_STATUS': HVAC = str(msg.payload) def on_subscribe(mosq, obj, mid, granted_qos): print("Subscribed: " + str(mid) + " " + str(granted_qos))
class HotQueueTestCase(unittest.TestCase): def setUp(self): """Create the queue instance before the test.""" self.queue = HotQueue("testqueue") def tearDown(self): """Clear the queue after the test.""" self.queue.clear() def test_arguments(self): """Test that HotQueue.__init__ accepts arguments correctly, and that the Redis key is correctly formed. """ kwargs = {"name": "testqueue", "serializer": DummySerializer, "host": "localhost", "port": 6379, "db": 0} # Instantiate the HotQueue instance: self.queue = HotQueue(**kwargs) # Ensure that the properties of the instance are as expected: self.assertEqual(self.queue.name, kwargs["name"]) self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs["name"]) self.assertEqual(self.queue.serializer, kwargs["serializer"]) self.assertEqual(self.queue._HotQueue__redis.host, kwargs["host"]) self.assertEqual(self.queue._HotQueue__redis.host, kwargs["host"]) self.assertEqual(self.queue._HotQueue__redis.port, kwargs["port"]) self.assertEqual(self.queue._HotQueue__redis.db, kwargs["db"]) # Instantiate a HotQueue instance with only the required args: self.queue = HotQueue(kwargs["name"]) # Ensure that the properties of the instance are as expected: self.assertEqual(self.queue.name, kwargs["name"]) self.assertEqual(self.queue.key, "hotqueue:%s" % kwargs["name"]) self.assertTrue(self.queue.serializer is pickle) # Defaults to cPickle # or pickle, depending # on the platform. def test_consume(self): """Test the consume generator method.""" nums = [1, 2, 3, 4, 5, 6, 7, 8] # Test blocking with timeout: self.queue.put(*nums) msgs = [] for msg in self.queue.consume(timeout=1): msgs.append(msg) self.assertEqual(msgs, nums) # Test non-blocking: self.queue.put(*nums) msgs = [] for msg in self.queue.consume(block=False): msgs.append(msg) self.assertEqual(msgs, nums) def test_cleared(self): """Test for correct behaviour if the Redis list does not exist.""" self.assertEqual(len(self.queue), 0) self.assertEqual(self.queue.get(), None) def test_get_order(self): """Test that messages are get in the same order they are put.""" alphabet = ["abc", "def", "ghi", "jkl", "mno"] self.queue.put(alphabet[0], alphabet[1], alphabet[2]) self.queue.put(alphabet[3]) self.queue.put(alphabet[4]) msgs = [] msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) self.assertEqual(msgs, alphabet) def test_length(self): """Test that the length of a queue is returned correctly.""" self.queue.put("a message") self.queue.put("another message") self.assertEqual(len(self.queue), 2) def test_worker(self): """Test the worker decorator.""" colors = ["blue", "green", "red", "pink", "black"] # Test blocking with timeout: self.queue.put(*colors) msgs = [] @self.queue.worker(timeout=1) def appender(msg): msgs.append(msg) appender() self.assertEqual(msgs, colors) # Test non-blocking: self.queue.put(*colors) msgs = [] @self.queue.worker(block=False) def appender(msg): msgs.append(msg) appender() self.assertEqual(msgs, colors) # Test decorating a class method: self.queue.put(*colors) msgs = [] class MyClass(object): @self.queue.worker(block=False) def appender(self, msg): msgs.append(msg) my_instance = MyClass() my_instance.appender() self.assertEqual(msgs, colors) def test_threaded(self): """Threaded test of put and consume methods.""" msgs = [] def put(): for num in range(3): self.queue.put("message %d" % num) sleep(0.1) def consume(): for msg in self.queue.consume(timeout=1): msgs.append(msg) putter = threading.Thread(target=put) consumer = threading.Thread(target=consume) putter.start() consumer.start() for thread in [putter, consumer]: thread.join() self.assertEqual(msgs, ["message 0", "message 1", "message 2"]) def test_custom_serializer(self): """Test the use of a custom serializer.""" self.queue.serializer = DummySerializer phrase = "my message" self.queue.put(phrase) msg = self.queue.get() self.assertEqual(msg, phrase)
class HotQueueTestCase(unittest.TestCase): def setUp(self): """Create the queue instance before the test.""" self.queue = HotQueue('testqueue') def tearDown(self): """Clear the queue after the test.""" self.queue.clear() def test_consume(self): """Test the consume generator method.""" nums = [1, 2, 3, 4, 5, 6, 7, 8] # Test blocking with timeout: self.queue.put(*nums) msgs = [] for msg in self.queue.consume(timeout=1): msgs.append(msg) self.assertEquals(msgs, nums) # Test non-blocking: self.queue.put(*nums) msgs = [] for msg in self.queue.consume(block=False): msgs.append(msg) self.assertEquals(msgs, nums) def test_cleared(self): """Test for correct behaviour if the Redis list does not exist.""" self.assertEquals(len(self.queue), 0) self.assertEquals(self.queue.get(), None) def test_get_order(self): """Test that messages are get in the same order they are put.""" alphabet = ['abc', 'def', 'ghi', 'jkl', 'mno'] self.queue.put(alphabet[0], alphabet[1], alphabet[2]) self.queue.put(alphabet[3]) self.queue.put(alphabet[4]) msgs = [] msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) msgs.append(self.queue.get()) self.assertEquals(msgs, alphabet) def test_length(self): """Test that the length of a queue is returned correctly.""" self.queue.put('a message') self.queue.put('another message') self.assertEquals(len(self.queue), 2) def test_worker(self): """Test the worker decorator.""" colors = ['blue', 'green', 'red', 'pink', 'black'] # Test blocking with timeout: self.queue.put(*colors) msgs = [] @self.queue.worker(timeout=1) def appender(msg): msgs.append(msg) appender() self.assertEqual(msgs, colors) # Test non-blocking: self.queue.put(*colors) msgs = [] @self.queue.worker(block=False) def appender(msg): msgs.append(msg) appender() self.assertEqual(msgs, colors) def test_threaded(self): """Threaded test of put and consume methods.""" msgs = [] def put(): for num in range(3): self.queue.put('message %d' % num) sleep(0.1) def consume(): for msg in self.queue.consume(timeout=1): msgs.append(msg) putter = threading.Thread(target=put) consumer = threading.Thread(target=consume) putter.start() consumer.start() for thread in [putter, consumer]: thread.join() self.assertEqual(msgs, ['message 0', 'message 1', 'message 2'])