def dequeue_worker(worker_id): global STOP_THREADS, deqs, acks, no_acks sq = SharQ('./sharq.conf') while STOP_THREADS is False: response = sq.dequeue(queue_type='sms') if len(response) <= 1: continue status = response.get('status', 'failure') if status == 'success': # make a note of when a job was dequeued deq_time[response['job_id']].append(datetime.now()) # increment dequeue counter for the job deqs[response['job_id']] += 1 # randomly call finish or not if bool(random.getrandbits(1)): sq.finish(queue_type='sms', job_id=response['job_id'], queue_id=response['queue_id']) acks[response['job_id']] += 1 else: no_acks[response['job_id']] += 1 print("thread-{}: exiting".format(worker_id))
def requeue_worker(worker_id): sq = SharQ('./sharq.conf') while STOP_THREADS is False: try: with sq.redis_client().lock('sharq-requeue-lock-key', timeout=15): try: sq.requeue() except Exception as e: traceback.print_exc() except LockError: # the lock wasn't acquired within specified time pass finally: sleep(REQUEUE_INTERVAL) print("thread-{}: exiting".format(worker_id))
def setUp(self): cwd = os.path.dirname(os.path.realpath(__file__)) config_path = os.path.join(cwd, 'sharq.test.conf') # test config self.queue = SharQ(config_path) self.valid_queue_type = '5m5_qu-eue' self.invalid_queue_type_1 = 's!ms_queue' self.invalid_queue_type_2 = 's!ms queue' self.invalid_queue_type_3 = '' self.valid_queue_id = 'queue_001-' self.invalid_queue_id_1 = 'queue#002' self.invalid_queue_id_2 = 'queue 002' self.invalid_queue_id_3 = '' self.valid_job_id = '96c82500-9f88-11e3-bb98-22000ac6964a' self.invalid_job_id_1 = '93 c8' self.invalid_job_id_2 = '93)c8' self.invalid_job_id_2 = '' self.valid_interval = 5000 self.invalid_interval_1 = '100' self.invalid_interval_2 = '$#' self.invalid_interval_3 = '' self.invalid_interval_4 = 0 self.invalid_interval_5 = -1 self.valid_requeue_limit_1 = 5 self.valid_requeue_limit_2 = 0 self.valid_requeue_limit_3 = -1 self.invalid_requeue_limit_1 = '100' self.invalid_requeue_limit_2 = '$#' self.invalid_requeue_limit_3 = '' self.invalid_requeue_limit_4 = -2 self.valid_payload = { 'phone_number': '1000000000', 'message': 'hello world' } self.invalid_payload = { 'phone_number': '10000000000', 'message': 'summer is here!', 'date': date.today() } # flush redis before start self.queue._r.flushdb()
def __init__(self, config_path): """Load the SharQ config and define the routes.""" # read the configs required by sharq-server. self.config = ConfigParser.SafeConfigParser() self.config.read(config_path) # pass the config file to configure the SharQ core. self.sq = SharQ(config_path) self.app = Flask(__name__) # set the routes self.app.add_url_rule('/', view_func=self._view_index, methods=['GET']) self.app.add_url_rule('/enqueue/<queue_type>/<queue_id>/', view_func=self._view_enqueue, methods=['POST']) self.app.add_url_rule('/dequeue/', defaults={'queue_type': 'default'}, view_func=self._view_dequeue, methods=['GET']) self.app.add_url_rule('/dequeue/<queue_type>/', view_func=self._view_dequeue, methods=['GET']) self.app.add_url_rule('/finish/<queue_type>/<queue_id>/<job_id>/', view_func=self._view_finish, methods=['POST']) self.app.add_url_rule('/interval/<queue_type>/<queue_id>/', view_func=self._view_interval, methods=['POST']) self.app.add_url_rule('/metrics/', defaults={ 'queue_type': None, 'queue_id': None }, view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule('/metrics/<queue_type>/', defaults={'queue_id': None}, view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule('/metrics/<queue_type>/<queue_id>/', view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule('/deletequeue/<queue_type>/<queue_id>/', view_func=self._view_clear_queue, methods=['DELETE']) self.app.add_url_rule('/deepstatus/', view_func=self._view_deep_status, methods=['GET'])
def __init__(self, config_path): """Load the SharQ config and define the routes.""" # read the configs required by sharq-server. self.config = ConfigParser.SafeConfigParser() self.config.read(config_path) # pass the config file to configure the SharQ core. self.sq = SharQ(config_path) self.app = Flask(__name__) # set the routes self.app.add_url_rule( '/', view_func=self._view_index, methods=['GET']) self.app.add_url_rule( '/enqueue/<queue_type>/<queue_id>/', view_func=self._view_enqueue, methods=['POST']) self.app.add_url_rule( '/dequeue/', defaults={'queue_type': 'default'}, view_func=self._view_dequeue, methods=['GET']) self.app.add_url_rule( '/dequeue/<queue_type>/', view_func=self._view_dequeue, methods=['GET']) self.app.add_url_rule( '/finish/<queue_type>/<queue_id>/<job_id>/', view_func=self._view_finish, methods=['POST']) self.app.add_url_rule( '/interval/<queue_type>/<queue_id>/', view_func=self._view_interval, methods=['POST']) self.app.add_url_rule( '/metrics/', defaults={'queue_type': None, 'queue_id': None}, view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule( '/metrics/<queue_type>/', defaults={'queue_id': None}, view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule( '/metrics/<queue_type>/<queue_id>/', view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule( '/deletequeue/<queue_type>/<queue_id>/', view_func=self._view_clear_queue, methods=['DELETE'])
class SharQTest(unittest.TestCase): """ The SharQTest contains test cases which validate the SharQ interface. """ def setUp(self): cwd = os.path.dirname(os.path.realpath(__file__)) config_path = os.path.join(cwd, 'sharq.test.conf') # test config self.queue = SharQ(config_path) self.valid_queue_type = '5m5_qu-eue' self.invalid_queue_type_1 = 's!ms_queue' self.invalid_queue_type_2 = 's!ms queue' self.invalid_queue_type_3 = '' self.valid_queue_id = 'queue_001-' self.invalid_queue_id_1 = 'queue#002' self.invalid_queue_id_2 = 'queue 002' self.invalid_queue_id_3 = '' self.valid_job_id = '96c82500-9f88-11e3-bb98-22000ac6964a' self.invalid_job_id_1 = '93 c8' self.invalid_job_id_2 = '93)c8' self.invalid_job_id_2 = '' self.valid_interval = 5000 self.invalid_interval_1 = '100' self.invalid_interval_2 = '$#' self.invalid_interval_3 = '' self.invalid_interval_4 = 0 self.invalid_interval_5 = -1 self.valid_requeue_limit_1 = 5 self.valid_requeue_limit_2 = 0 self.valid_requeue_limit_3 = -1 self.invalid_requeue_limit_1 = '100' self.invalid_requeue_limit_2 = '$#' self.invalid_requeue_limit_3 = '' self.invalid_requeue_limit_4 = -2 self.valid_payload = { 'phone_number': '1000000000', 'message': 'hello world' } self.invalid_payload = { 'phone_number': '10000000000', 'message': 'summer is here!', 'date': date.today() } # flush redis before start self.queue._r.flushdb() def test_enqueue_queue_type_invalid(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.invalid_queue_type_1 ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.invalid_queue_type_2 ) # type 3 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.invalid_queue_type_3 ) def test_enqueue_queue_id_missing(self): self.assertRaises( TypeError, self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, # queue_id is missing queue_type=self.valid_queue_type ) def test_enqueue_queue_id_invalid(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.invalid_queue_id_1, queue_type=self.valid_queue_type ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.invalid_queue_id_2, queue_type=self.valid_queue_type ) # type 3 self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.invalid_queue_id_3, queue_type=self.valid_queue_type ) def test_enqueue_job_id_missing(self): self.assertRaises( TypeError, self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, # job_id is missing queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) def test_enqueue_job_id_invalid(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`job_id` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.invalid_job_id_1, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`job_id` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.invalid_job_id_2, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) def test_enqueue_interval_missing(self): self.assertRaises( TypeError, self.queue.enqueue, payload=self.valid_payload, # interval is missing job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) def test_enqueue_interval_invalid(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`interval` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.invalid_interval_1, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`interval` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.invalid_interval_2, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) # type 3 self.assertRaisesRegexp( BadArgumentException, '`interval` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.invalid_interval_3, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) # type 4 self.assertRaisesRegexp( BadArgumentException, '`interval` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.invalid_interval_4, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) # type 5 self.assertRaisesRegexp( BadArgumentException, '`interval` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.invalid_interval_5, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) def test_enqueue_requeue_limit_invalid(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`requeue_limit` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type, requeue_limit=self.invalid_requeue_limit_1 ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`requeue_limit` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type, requeue_limit=self.invalid_requeue_limit_2 ) # type 3 self.assertRaisesRegexp( BadArgumentException, '`requeue_limit` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type, requeue_limit=self.invalid_requeue_limit_3 ) # type 4 self.assertRaisesRegexp( BadArgumentException, '`requeue_limit` has an invalid value.', self.queue.enqueue, payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type, requeue_limit=self.invalid_requeue_limit_4 ) def test_enqueue_cannot_serialize_payload(self): self.assertRaisesRegexp( BadArgumentException, r'can\'t serialize.', self.queue.enqueue, payload=self.invalid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) def test_enqueue_all_ok(self): # with a queue_type response = self.queue.enqueue( payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) self.assertEqual(response['status'], 'queued') # the result should contain only status response.pop('status') self.assertEqual(response, {}) # without a queue_type (queue_type will be 'default') response = self.queue.enqueue( payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id ) self.assertEqual(response['status'], 'queued') # the result should contain only status response.pop('status') self.assertEqual(response, {}) # with requeue_limit 1 response = self.queue.enqueue( payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type, requeue_limit=self.valid_requeue_limit_1 ) self.assertEqual(response['status'], 'queued') # the result should contain only status response.pop('status') self.assertEqual(response, {}) # with requeue_limit 2 response = self.queue.enqueue( payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type, requeue_limit=self.valid_requeue_limit_2 ) self.assertEqual(response['status'], 'queued') # the result should contain only status response.pop('status') self.assertEqual(response, {}) # with requeue_limit 3 response = self.queue.enqueue( payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type, requeue_limit=self.valid_requeue_limit_3 ) self.assertEqual(response['status'], 'queued') # the result should contain only status response.pop('status') self.assertEqual(response, {}) # requeue_limit missing response = self.queue.enqueue( payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) self.assertEqual(response['status'], 'queued') # the result should contain only status response.pop('status') self.assertEqual(response, {}) def test_dequeue_queue_type_invalid(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.dequeue, queue_type=self.invalid_queue_type_1 ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.dequeue, queue_type=self.invalid_queue_type_2 ) # type 3 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.dequeue, queue_type=self.invalid_queue_type_3 ) def test_dequeue_all_ok(self): # first enqueue a job self.queue.enqueue( payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) # with a queue_type response = self.queue.dequeue( queue_type=self.valid_queue_type ) self.assertEqual(response['status'], 'success') response.pop('status') # check if it has a key called 'payload' self.assertIn('payload', response) response.pop('payload') # check if it has a key called 'queue_id' self.assertIn('queue_id', response) response.pop('queue_id') # check if it has a key called 'job_id' self.assertIn('job_id', response) response.pop('job_id') # check if it has a key called 'requeues_remaining' self.assertIn('requeues_remaining', response) response.pop('requeues_remaining') # make sure nothing else in response # except the above key / value pairs self.assertEqual(response, {}) # enqueue another job self.queue.enqueue( payload=self.valid_payload, interval=self.valid_interval, job_id=self.valid_job_id, queue_id=self.valid_queue_id ) # without a queue_type response = self.queue.dequeue() self.assertEqual(response['status'], 'success') response.pop('status') # check if it has a key called 'payload' self.assertIn('payload', response) response.pop('payload') # check if it has a key called 'queue_id' self.assertIn('queue_id', response) response.pop('queue_id') # check if it has a key called 'job_id' self.assertIn('job_id', response) response.pop('job_id') # check if it has a key called 'requeues_remaining' self.assertIn('requeues_remaining', response) response.pop('requeues_remaining') # make sure nothing else in response # except the above key / value pairs self.assertEqual(response, {}) def test_finish_queue_type_invalid(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.finish, queue_type=self.invalid_queue_type_1, queue_id=self.valid_queue_id, job_id=self.valid_job_id ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.finish, queue_type=self.invalid_queue_type_2, queue_id=self.valid_queue_id, job_id=self.valid_job_id ) # type 3 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.finish, queue_type=self.invalid_queue_type_3, queue_id=self.valid_queue_id, job_id=self.valid_job_id ) def test_finish_queue_id_missing(self): self.assertRaises( TypeError, self.queue.finish, queue_type=self.valid_queue_type, # queue_id missing job_id=self.valid_job_id ) def test_finish_queue_id_invalid(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.finish, queue_type=self.valid_queue_type, queue_id=self.invalid_queue_id_1, job_id=self.valid_job_id ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.finish, queue_type=self.valid_queue_type, queue_id=self.invalid_queue_id_2, job_id=self.valid_job_id ) # type 3 self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.finish, queue_type=self.valid_queue_type, queue_id=self.invalid_queue_id_3, job_id=self.valid_job_id ) def test_finish_job_id_missing(self): self.assertRaises( TypeError, self.queue.finish, queue_type=self.valid_queue_type, queue_id=self.valid_queue_id, # job_id missing ) def test_finish_job_id_invalid(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`job_id` has an invalid value.', self.queue.finish, queue_type=self.valid_queue_type, queue_id=self.valid_queue_id, job_id=self.invalid_job_id_1 ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`job_id` has an invalid value.', self.queue.finish, queue_type=self.valid_queue_type, queue_id=self.valid_queue_id, job_id=self.invalid_job_id_2 ) def test_finish_all_ok(self): # with a queue_type. no existent job. response = self.queue.finish( queue_type=self.valid_queue_type, queue_id=self.valid_queue_id, job_id=self.valid_job_id ) self.assertEqual(response['status'], 'failure') response.pop('status') # make sure nothing else in response # except the above key / value pairs self.assertEqual(response, {}) # without a queue_type response = self.queue.finish( queue_id=self.valid_queue_id, job_id=self.valid_job_id ) self.assertEqual(response['status'], 'failure') response.pop('status') # make sure nothing else in response # except the above key / value pairs self.assertEqual(response, {}) def test_interval_interval_invalid(self): self.assertRaisesRegexp( BadArgumentException, '`interval` has an invalid value.', self.queue.interval, interval=self.invalid_interval_1, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) self.assertRaisesRegexp( BadArgumentException, '`interval` has an invalid value.', self.queue.interval, interval=self.invalid_interval_2, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) self.assertRaisesRegexp( BadArgumentException, '`interval` has an invalid value.', self.queue.interval, interval=self.invalid_interval_3, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) self.assertRaisesRegexp( BadArgumentException, '`interval` has an invalid value.', self.queue.interval, interval=self.invalid_interval_4, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) self.assertRaisesRegexp( BadArgumentException, '`interval` has an invalid value.', self.queue.interval, interval=self.invalid_interval_5, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) def test_interval_interval_missing(self): self.assertRaises( TypeError, self.queue.interval, # interval parameter missing queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) def test_interval_invalid_queue_id(self): self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.interval, interval=self.valid_interval, queue_id=self.invalid_queue_id_1, queue_type=self.valid_queue_type ) self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.interval, interval=self.valid_interval, queue_id=self.invalid_queue_id_2, queue_type=self.valid_queue_type ) self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.interval, interval=self.valid_interval, queue_id=self.invalid_queue_id_3, queue_type=self.valid_queue_type ) def test_interval_queue_id_missing(self): self.assertRaises( TypeError, self.queue.interval, interval=self.valid_interval, # queue_id parameter missing queue_type=self.valid_queue_type ) def test_interval_invalid_queue_type(self): self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.interval, interval=self.valid_interval, queue_id=self.valid_queue_id, queue_type=self.invalid_queue_type_1 ) self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.interval, interval=self.valid_interval, queue_id=self.valid_queue_id, queue_type=self.invalid_queue_type_2 ) self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.interval, interval=self.valid_interval, queue_id=self.valid_queue_id, queue_type=self.invalid_queue_type_3 ) def test_interval_all_ok(self): # with a queue_type response = self.queue.interval( interval=self.valid_interval, queue_id=self.valid_queue_id, queue_type=self.valid_queue_type ) # no queues are found yet. self.assertEqual(response['status'], 'failure') # the result should contain only status response.pop('status') self.assertEqual(response, {}) # without a queue_type response = self.queue.interval( interval=self.valid_interval, queue_id=self.valid_queue_id ) # no queues are found yet. self.assertEqual(response['status'], 'failure') # the result should contain only status response.pop('status') self.assertEqual(response, {}) def test_metrics_no_argument(self): # with no arguments response = self.queue.metrics() self.assertEqual(response['status'], 'success') response.pop('status') # check if it has a key called 'queue_types' self.assertIn('queue_types', response) response.pop('queue_types') # check if it has a key called 'enqueue_counts' self.assertIn('enqueue_counts', response) response.pop('enqueue_counts') # check if it has a key called 'dequeue_counts' self.assertIn('dequeue_counts', response) response.pop('dequeue_counts') self.assertEqual(response, {}) def test_metrics_only_queue_id(self): # with only a valid queue_id self.assertRaisesRegexp( BadArgumentException, '`queue_id` should be accompanied by `queue_type`.', self.queue.metrics, queue_id=self.valid_queue_id, ) def test_metrics_only_queue_type(self): # with only a valid queue_type response = self.queue.metrics(queue_type=self.valid_queue_type) self.assertEqual(response['status'], 'success') response.pop('status') # check if it has a key called 'queue_ids' self.assertIn('queue_ids', response) response.pop('queue_ids') self.assertEqual(response, {}) def test_metrics_both_queue_id_queue_type(self): # with a valid queue_id and queue_type response = self.queue.metrics( queue_type=self.valid_queue_type, queue_id=self.valid_queue_id) self.assertEqual(response['status'], 'success') response.pop('status') # check if it has a key called 'queue_length' self.assertIn('queue_length', response) response.pop('queue_length') # check if it has a key called 'enqueue_counts' self.assertIn('enqueue_counts', response) response.pop('enqueue_counts') # check if it has a key called 'dequeue_counts' self.assertIn('dequeue_counts', response) response.pop('dequeue_counts') self.assertEqual(response, {}) def test_metrics_queue_id_invalid(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.metrics, queue_type=self.valid_queue_type, queue_id=self.invalid_queue_id_1 ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.metrics, queue_type=self.valid_queue_type, queue_id=self.invalid_queue_id_2 ) # type 3 self.assertRaisesRegexp( BadArgumentException, '`queue_id` has an invalid value.', self.queue.metrics, queue_type=self.valid_queue_type, queue_id=self.invalid_queue_id_3 ) def test_metrics_invalid_queue_type(self): # type 1 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.metrics, queue_type=self.invalid_queue_type_1, queue_id=self.valid_queue_id ) # type 2 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.metrics, queue_type=self.invalid_queue_type_2, queue_id=self.valid_queue_id ) # type 3 self.assertRaisesRegexp( BadArgumentException, '`queue_type` has an invalid value.', self.queue.metrics, queue_type=self.invalid_queue_type_3, queue_id=self.valid_queue_id ) def tearDown(self): # flush redis at the end self.queue._r.flushdb()
class SharQServer(object): """Defines a HTTP based API on top of SharQ and exposed the app to run the server. """ def __init__(self, config_path): """Load the SharQ config and define the routes.""" # read the configs required by sharq-server. self.config = ConfigParser.SafeConfigParser() self.config.read(config_path) # pass the config file to configure the SharQ core. self.sq = SharQ(config_path) self.app = Flask(__name__) # set the routes self.app.add_url_rule( '/', view_func=self._view_index, methods=['GET']) self.app.add_url_rule( '/enqueue/<queue_type>/<queue_id>/', view_func=self._view_enqueue, methods=['POST']) self.app.add_url_rule( '/dequeue/', defaults={'queue_type': 'default'}, view_func=self._view_dequeue, methods=['GET']) self.app.add_url_rule( '/dequeue/<queue_type>/', view_func=self._view_dequeue, methods=['GET']) self.app.add_url_rule( '/finish/<queue_type>/<queue_id>/<job_id>/', view_func=self._view_finish, methods=['POST']) self.app.add_url_rule( '/interval/<queue_type>/<queue_id>/', view_func=self._view_interval, methods=['POST']) self.app.add_url_rule( '/metrics/', defaults={'queue_type': None, 'queue_id': None}, view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule( '/metrics/<queue_type>/', defaults={'queue_id': None}, view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule( '/metrics/<queue_type>/<queue_id>/', view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule( '/deletequeue/<queue_type>/<queue_id>/', view_func=self._view_clear_queue, methods=['DELETE']) def requeue(self): """Loop endlessly and requeue expired jobs.""" job_requeue_interval = float( self.config.get('sharq', 'job_requeue_interval')) while True: self.sq.requeue() gevent.sleep(job_requeue_interval / 1000.00) # in seconds def _view_index(self): """Greetings at the index.""" return jsonify(**{'message': 'Hello, SharQ!'}) def _view_enqueue(self, queue_type, queue_id): """Enqueues a job into SharQ.""" response = { 'status': 'failure' } try: request_data = json.loads(request.data) except Exception, e: response['message'] = e.message return jsonify(**response), 400 request_data.update({ 'queue_type': queue_type, 'queue_id': queue_id }) try: response = self.sq.enqueue(**request_data) except Exception, e: response['message'] = e.message return jsonify(**response), 400
class SharQServer(object): """Defines a HTTP based API on top of SharQ and exposed the app to run the server. """ def __init__(self, config_path): """Load the SharQ config and define the routes.""" # read the configs required by sharq-server. self.config = ConfigParser.SafeConfigParser() self.config.read(config_path) # pass the config file to configure the SharQ core. self.sq = SharQ(config_path) self.app = Flask(__name__) # set the routes self.app.add_url_rule('/', view_func=self._view_index, methods=['GET']) self.app.add_url_rule('/enqueue/<queue_type>/<queue_id>/', view_func=self._view_enqueue, methods=['POST']) self.app.add_url_rule('/dequeue/', defaults={'queue_type': 'default'}, view_func=self._view_dequeue, methods=['GET']) self.app.add_url_rule('/dequeue/<queue_type>/', view_func=self._view_dequeue, methods=['GET']) self.app.add_url_rule('/finish/<queue_type>/<queue_id>/<job_id>/', view_func=self._view_finish, methods=['POST']) self.app.add_url_rule('/interval/<queue_type>/<queue_id>/', view_func=self._view_interval, methods=['POST']) self.app.add_url_rule('/metrics/', defaults={ 'queue_type': None, 'queue_id': None }, view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule('/metrics/<queue_type>/', defaults={'queue_id': None}, view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule('/metrics/<queue_type>/<queue_id>/', view_func=self._view_metrics, methods=['GET']) self.app.add_url_rule('/deletequeue/<queue_type>/<queue_id>/', view_func=self._view_clear_queue, methods=['DELETE']) self.app.add_url_rule('/deepstatus/', view_func=self._view_deep_status, methods=['GET']) def requeue(self): """Loop endlessly and requeue expired jobs.""" job_requeue_interval = float( self.config.get('sharq', 'job_requeue_interval')) while True: self.sq.requeue() gevent.sleep(job_requeue_interval / 1000.00) # in seconds def _view_index(self): """Greetings at the index.""" return jsonify(**{'message': 'Hello, SharQ!'}) def _view_enqueue(self, queue_type, queue_id): """Enqueues a job into SharQ.""" response = {'status': 'failure'} try: request_data = json.loads(request.data) except Exception, e: response['message'] = e.message return jsonify(**response), 400 request_data.update({'queue_type': queue_type, 'queue_id': queue_id}) try: response = self.sq.enqueue(**request_data) except Exception, e: response['message'] = e.message return jsonify(**response), 400
from sharq import SharQ sq = SharQ('./sharq.conf') for i in range(1000): resp = sq.enqueue( job_id=str(i), payload={'a': 'b'}, interval=10, # in milliseconds. queue_id='user001', queue_type='sms')