class Queue:
def __init__(self, conf):
self.conf = conf
self.client = Client([':'.join([self.host, str(self.port)])])
self.client.connect()
@property
def host(self):
return self.conf.get('host') or 'localhost'
@property
def port(self):
return self.conf.get('port') or 7711
@property
def queue(self):
return self.conf.get('queue') or 'downloader'
def get(self):
return self.client.get_job([self.queue])
def add(self, job):
return self.client.add_job(self.queue, json.dumps(job), timeout=1000)
def ack(self, job_id):
return self.client.ack_job(job_id)
def init(disque_servers=(("localhost", 7711), )):
disque_servers = [
"%s:%d" % disque_server for disque_server in disque_servers
]
logger.info('Connecting to disque servers: %s' % ', '.join(disque_servers))
global disque_client
disque_client = DisqueClient(disque_servers)
disque_client.connect()
def handle(self, *args, **options):
# get a list of all Topics
queues = {}
topics = Topic.objects.all()
for topic in topics:
queues[topic.topic] = topic.id
c = Client(servers)
c.connect()
# listen forever
print(queues.keys())
while(True):
jobs = c.get_job(list(queues.keys()))
for queue_name, job_id, job in jobs:
try:
topic = Topic.objects.get(topic=queue_name)
new_message = Message(topic=topic, body=job, ack_date=timezone.now())
new_message.save()
c.ack_job(job_id)
except Exception as e:
c.nack_job(job_id)
time.sleep(1)
def main():
"""Start the poor_consumer."""
try:
opts, args = getopt.getopt(sys.argv[1:], "h:v",
["help", "nack=", "servers=", "queues="])
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit()
# defaults
nack = 0.0
verbose = False
servers = "localhost:7712,localhost:7711"
queues = "test"
for o, a in opts:
if o == "-v":
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("--nack"):
nack = float(a)
elif o in ("--servers"):
servers = a
elif o in ("--queues"):
queues = a
else:
assert False, "unhandled option"
# prepare servers and queus for pydisque
servers = servers.split(",")
queues = queues.split(",")
c = Client(servers)
c.connect()
while True:
jobs = c.get_job(queues)
for queue_name, job_id, job in jobs:
rnd = random.random()
# as this is a test processor, we don't do any validation on
# the actual job body, so lets just pay attention to id's
if rnd >= nack:
print ">>> received job:", job_id
c.ack_job(job_id)
else:
print ">>> bouncing job:", job_id
c.nack_job(job_id)
def main():
"""Start the poor_consumer."""
try:
opts, args = getopt.getopt(sys.argv[1:], "h:v", ["help", "nack=",
"servers=", "queues="])
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit()
# defaults
nack = 0.0
verbose = False
servers = "localhost:7712,localhost:7711"
queues = "test"
for o, a in opts:
if o == "-v":
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("--nack"):
nack = float(a)
elif o in ("--servers"):
servers = a
elif o in ("--queues"):
queues = a
else:
assert False, "unhandled option"
# prepare servers and queus for pydisque
servers = servers.split(",")
queues = queues.split(",")
c = Client(servers)
c.connect()
while True:
jobs = c.get_job(queues)
for queue_name, job_id, job in jobs:
rnd = random.random()
# as this is a test processor, we don't do any validation on
# the actual job body, so lets just pay attention to id's
if rnd >= nack:
print ">>> received job:", job_id
c.ack_job(job_id)
else:
print ">>> bouncing job:", job_id
c.nack_job(job_id)
def setUp(self):
"""Setup the tests."""
self.client = Client(['localhost:7711'])
self.client.connect()
self.testID = "%d.%d" % (time.time(),
random.randint(1000, 1000000))
class TestDisque(unittest.TestCase):
"""TestCase class for pydisque."""
testID = None
def setUp(self):
"""Setup the tests."""
self.client = Client(['localhost:7711'])
self.client.connect()
self.testID = "%d.%d" % (time.time(),
random.randint(1000, 1000000))
def test_publish_and_receive(self):
"""Test the most important functions of pydisque."""
t1 = str(time.time())
self.client.add_job("test_q", t1, timeout=100)
jobs = self.client.get_job(['test_q'])
assert len(jobs) == 1
for queue_name, job_id, job in jobs:
assert job == six.b(t1)
self.client.ack_job(job_id)
assert len(self.client.get_job(['test_q'], timeout=100)) == 0
def test_nack(self):
"""Fetch the queue, return a job, check that it's back."""
t1 = str(time.time())
queuename = "test_nack." + self.testID
self.client.add_job(queuename, str(t1), timeout=100)
jobs = self.client.get_job([queuename])
# NACK the first read
assert len(jobs) == 1
for queue_name, job_id, job in jobs:
assert len(jobs) == 1
assert job == six.b(t1)
self.client.nack_job(job_id)
# this time ACK it
jobs = self.client.get_job([queuename])
assert len(jobs) == 1
for queue_name, job_id, job in jobs:
assert job == six.b(t1)
self.client.ack_job(job_id)
assert len(self.client.get_job([queuename], timeout=100)) == 0
def test_qpeek(self):
"""
Test qpeek.
Ran into some problems with an ENQUEUE/DEQUEUE test that
was using qpeek, checking core functionality of qpeek().
"""
queuename = "test_qpeek-%s" % self.testID
job_id = self.client.add_job(queuename, "Peek A Boo")
peeked = self.client.qpeek(queuename, 1)
assert peeked[0][1] == job_id
def test_qscan(self):
"""
Test the qscan function.
This test relies on add_job() being functional, and
the local disque not being a disque proxy to a mesh.
TODO: unique the queues with self.testID.
"""
t1 = str(time.time())
self.client.add_job("q1", t1, timeout=100)
self.client.add_job("q2", t1, timeout=100)
qb = self.client.qscan()
assert qb[0]
assert qb[1]
assert six.b("q1") in qb[1]
assert six.b("q2") in qb[1]
def test_jscan(self):
"""Simple test of the jscan function."""
t1 = time.time()
queuename = "test_jscan-%s" % self.testID
j1 = self.client.add_job(queuename, str(t1), timeout=100)
jerbs = self.client.jscan(queue=queuename)
assert j1 in jerbs[1]
def test_del_job(self):
"""Simple test of del_job, needs qpeek.
FIXME: This function has grown ugly.
"""
t1 = time.time()
queuename = "test_del_job-%s" % self.testID
j1 = self.client.add_job(queuename, str(t1))
jerbs = self.client.qpeek(queuename, 1)
jlist = []
for item in jerbs:
jlist.append(item[1])
assert j1 in jlist
self.client.del_job(j1)
jerbs = self.client.qpeek(queuename, 1)
jlist = []
for item in jerbs:
jlist.append(item[1])
assert j1 not in jerbs
def test_qlen(self):
"""Simple test of qlen."""
queuename = "test_qlen-%s" % self.testID
lengthOfTest = 100
test_job = "Useless Job."
for x in range(lengthOfTest):
self.client.add_job(queuename, test_job)
assert self.client.qlen(queuename) == lengthOfTest
def test_qstat(self):
"""Testing QSTAT (default behavior)."""
queuename = "test_qstat-%s" % self.testID
testqueue = ["a", "b", "c"]
for x in testqueue:
self.client.add_job(queuename, x)
stat = self.client.qstat(queuename)
# check the basics
assert b'jobs-in' in stat
assert b'jobs-out' in stat
def test_qstat_dict(self):
"""Testing QSTAT's (new dict behavior)."""
queuename = "test_qstat_dict-%s" % self.testID
testqueue = ["a", "b", "c"]
for x in testqueue:
self.client.add_job(queuename, x)
stat = self.client.qstat(queuename, True)
assert stat.get(b'jobs-in', None) is not None
assert stat.get(b'jobs-out', None) is not None
def test_shownack(self):
"""Test that NACK and SHOW work appropriately."""
queuename = "test_show-%s" % self.testID
test_job = "Show me."
self.client.add_job(queuename, test_job)
jobs = self.client.get_job([queuename])
for queue_name, job_id, job in jobs:
self.client.nack_job(job_id)
shown = self.client.show(job_id, True)
assert shown.get(b'body') == b(test_job)
assert shown.get(b'nacks') == 1
def _test_pause(self):
""" TODO: """
"""Test that a PAUSE message is acknowledged."""
queuename = "test_show-%s" % self.testID
test_job = "Jerbs, they are a thing"
self.client.pause(queuename, kw_in=True)
try:
job_id = self.client.add_job(queuename, test_job)
except ResponseError:
pass
# can we add a job again?
self.client.pause(queuename, kw_none=True)
job_id = self.client.add_job(queuename, test_job)
jobs = self.client.get_job([queuename])
# TODO(canardleteer): add a test of PAUSE SHOW
def test_get_job(self):
queue_name = "test_get_job." + self.testID
job = str(time.time())
job_id = self.client.add_job(queue_name, job)
expected = [(b(queue_name), job_id, b(job))]
got = self.client.get_job([queue_name], withcounters=False)
assert expected == got
def test_get_job_withcounters(self):
queue_name = "test_get_job." + self.testID
job = str(time.time())
job_id = self.client.add_job(queue_name, job)
nacks = 0
additional_deliveries = 0
expected = [(b(queue_name), job_id, b(job), nacks, additional_deliveries)]
got = self.client.get_job([queue_name], withcounters=True)
assert expected == got
def main():
'''
This is the main method, validate args, load credentials, start the daemon.
'''
message = 'Listen to tweets; dump them to the queue.'
socket_help = ('a list containing the host, port numbers to listen to; '
'defaults to localhost:7711 (for disque)')
parser = ArgumentParser(description=message)
parser.add_argument('-s',
'--sockets',
help=socket_help,
default=['localhost:7711'],
dest='sockets',
metavar=('HOST:PORT'),
nargs='+')
parser.add_argument('-c',
'--channels',
help='Twitter accounts to follow',
dest='channels',
metavar=('CHANNEL'),
nargs='+',
required=True)
parser.add_argument('-d',
'--debug',
help='enable debugging',
action='store_true',
default=False)
args = vars(parser.parse_args())
if args['debug']:
LOGGER.setLevel(DEBUG)
LOGGER.addHandler(HANDLER)
else:
LOGGER.setLevel(INFO)
LOGGER.addHandler(HANDLER)
try:
# Connect to the redis-queue.
queue = Client(args['sockets'])
queue.connect()
LOGGER.info('[start-daemon]')
queue_info = json.dumps(queue.info(), indent=4)
LOGGER.debug('[queue-init]\n%s', queue_info)
# Load credentials, initialize authentication module, listen to tweets.
api = load_credentials()
if not api:
LOGGER.error('[load_credentials] unable to load credentials!')
return
listener = StreamDaemon(queue)
streamer = tweepy.Stream(auth=api.auth, listener=listener)
args['channels'] = [re.sub('@', '', _) for _ in args['channels']]
streamer.userstream(track=args['channels'])
except Exception:
LOGGER.error('[error] unknown error')
LOGGER.error('[error] unable to connect to the redis-queue (disque)!')
except KeyboardInterrupt:
LOGGER.critical('[stop-daemon]')
return
def setUp(self):
"""Setup the tests."""
self.client = Client(['localhost:7711'])
self.client.connect()
#!/usr/bin/env python3
import pytest
import subprocess
import uuid
from pydisque.client import Client as PydisqueClient
from plutoid_kernel.utils import form_message
import json
import time
CLIENT_CHANNEL = str(uuid.uuid4())
disque_client = PydisqueClient()
disque_client.connect()
@pytest.fixture
def kernel_details_session_mode(scope="session"):
kernel_id = str(uuid.uuid4())
cmd = 'plutoidkernel --session-mode --kernel-id %s --ping-interval 2' % kernel_id
kernel_proc = subprocess.Popen(cmd.split(' '))
yield (kernel_id, kernel_proc)
if not kernel_proc.returncode:
kernel_proc.terminate()
# clear the disque queue
for queue_name, system_message_id, message in disque_client.get_job(
[CLIENT_CHANNEL], timeout=1000):
disque_client.ack_job(system_message_id)
def setUp(self):
"""Setup the tests."""
self.client = Client(['localhost:7711'])
self.client.connect()
self.testID = "%d.%d" % (time.time(), random.randint(1000, 1000000))
config = {
#'host': 'd1-redis-addfix.iqi3ba.ng.0001.apse1.cache.amazonaws.com',
#'host': 'data-prod-redis-002.iqi3ba.0001.apse1.cache.amazonaws.com',
'host': 'localhost',
'port': 6379,
'db': 0,
}
#for redis connection
r = redis.Redis(**config)
if not r.ping():
sentry_client.captureException(
message="settings.py: Failed to connect to redis",
extra={"error": 'ping failed'})
sys.exit()
## for disque connection
client = Client(["127.0.0.1:7711"])
#client = Client(["10.0.4.232:7711"])
try:
client.connect()
except Exception as disque_err:
sentry_client.captureException(
message="settings.py:Failed to connect to disque",
extra={"error": disque_err})
sys.exit()
catfight_input = config_parser.get("Queues", "catfight_input")
catfight_output = config_parser.get("Queues", "catfight_output")
def add_job(request):
user = request.user
ip = request.META['REMOTE_ADDR']
zone = request.POST['zone']
queue = request.POST['queue_name']
# timeout_ms = request.POST['timeout_ms']
# replicate = request.POST['replicate']
# retry_sec = request.POST['retry_sec']
# delay_sec = request.POST['delay_sec']
# ttl_sec = request.POST['ttl_sec']
jobs = request.POST.getlist('jobs', [])
# print user, env, queue, timeout_ms, replicate, retry_sec, delay_sec, ttl_sec
print user, zone, queue
print jobs
if not user.groups.filter(name__in=['admin', 'dba', 'disque']).exists():
logs(user, ip, 'add job: %s - %s' % (zone, queue), 'permission denied')
return HttpResponse(json.dumps({'errcode': 403}),
content_type=DEFAULT_CONTENT_TYPE)
try:
clusterInfo = ClusterInfo.objects.get(name=zone)
print clusterInfo.addr
except ClusterInfo.DoesNotExist:
logs(user, ip, 'add job: %s - %s' % (zone, queue),
'unknown disque zone: %s' % zone)
return HttpResponse(json.dumps({
'errcode': 400,
'msg': 'unknown disque zone:%s' % zone
}),
content_type=DEFAULT_CONTENT_TYPE)
except ClusterInfo.MultipleObjectsReturned:
logs(user, ip, 'add job: %s - %s' % (zone, queue),
'multi objects returned for zone: %s' % zone)
return HttpResponse(json.dumps({
'errcode':
400,
'msg':
'multi objects returned for zone:%s' % zone
}),
content_type=DEFAULT_CONTENT_TYPE)
except Exception as e:
print e
logs(user, ip, 'add job: %s - %s' % (zone, queue), str(e))
return HttpResponse(json.dumps({
'errcode': 400,
'msg': str(e)
}),
content_type=DEFAULT_CONTENT_TYPE)
if (not queue) or len(queue) == 0:
logs(user, ip, 'add job: %s - %s' % (zone, queue), 'empty queue name')
return HttpResponse(json.dumps({
'errcode': 400,
'msg': 'empty queue name'
}),
content_type=DEFAULT_CONTENT_TYPE)
if len(jobs) == 0:
logs(user, ip, 'add job: %s - %s' % (zone, queue), 'empty jobs')
return HttpResponse(json.dumps({
'errcode': 400,
'msg': 'empty jobs'
}),
content_type=DEFAULT_CONTENT_TYPE)
jobs = map(lambda x: x.encode('utf-8'), jobs)
jobIds = []
errJob = []
addr = clusterInfo.addr.split(',')
client = Client(addr)
client.connect()
for job in jobs:
try:
print job
jobId = client.add_job(queue, job)
# jobId = client.add_job(queue, job, timeout=timeout_ms, replicate=replicate, delay=delay_sec, retry=retry_sec, ttl=ttl_sec)
jobIds.append(jobId)
except Exception as e:
print e
errJob.append(job)
logs(user, ip, 'add job: %s - %s' % (zone, queue), 'success')
return HttpResponse(json.dumps({
'errcode': 200,
'jobIds': jobIds,
'failJobs': errJob
}),
content_type=DEFAULT_CONTENT_TYPE)
import json
import time
import logging
logging.basicConfig(level=logging.DEBUG)
from pydisque.client import Client
c = Client(['localhost:7712', 'localhost:7711'])
c.connect()
while True:
t = time.time()
print "sending job", t
c.add_job("test", json.dumps(["print", "hello", "world", t]), replicate=1, timeout=100)
time.sleep(2)
def __init__(self, conf):
self.conf = conf
self.client = Client([':'.join([self.host, str(self.port)])])
self.client.connect()
def main():
"""The job listening loop."""
try:
opts, args = getopt.getopt(sys.argv[1:], "h:v", ["help",
"servers=", "queues="])
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit()
# defaults
servers = "localhost:7712,localhost:7711"
queues = "twitpush"
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("--servers"):
servers = a
elif o in ("--queues"):
queues = a
else:
assert False, "unhandled option"
# prepare servers and queues for pydisque
servers = servers.split(",")
queues = queues.split(",")
c = Client(servers)
c.connect()
# prepare our tweepy instance
tweepy_auth = tweepy.OAuthHandler(tweepy_consumer_key,
tweepy_consumer_secret)
tweepy_auth.set_access_token(tweepy_access_token,
tweepy_access_token_secret)
tweepy_api = tweepy.API(tweepy_auth)
while True:
jobs = c.get_job(queues)
for queue_name, job_id, job in jobs:
try:
ts, command, payload = pickle.loads(job)
if command == "UPDATE":
tweepy_api.update_status(payload)
print("tweepy_api.update_status(%s)" % payload)
c.ack_job(job_id)
else:
# we dunno what to do
c.nack_job(job_id)
except Exception as e:
# we shouldn't nack jobs if they shouldn't be requeued
if 'message' in e[0][0]:
if e[0][0]['message'] == 'Status is a duplicate.':
c.ack_job(job_id)
continue
c.nack_job(job_id)
# exiting for now to avoid slamming the Twitter API
print(e)
sys.exit()
import json
import time
import logging
logging.basicConfig(level=logging.DEBUG)
from pydisque.client import Client
c = Client(['localhost:7712', 'localhost:7711'])
c.connect()
while True:
jobs = c.get_job(['test'])
for queue_name, job_id, job in jobs:
job = json.loads(job)
print ">>> received job:", job
c.ack_job(job_id)
def main():
"""Primary CLI application logic."""
try:
opts, args = getopt.getopt(sys.argv[1:], "h:v", ["help",
"dservers=",
"dqueue=",
"secret=",
"bserver=",
"hserver=",
"rserver=",
"rchannel=",
"bfiltername=",
"hllname=",
"mode=",
"sleep="])
except getopt.GetoptError as err:
print(str(err))
usage()
sys.exit()
modes = ("generate", "listen", "check", "adaptive", "initialize", "subscriber")
# set defaults
mode = None
dservers = "localhost:7712,localhost:7711"
dqueue = "objbomber"
secret = "coolsecretbro"
bserver = "localhost:8673"
hserver = "localhost:4553"
rserver = "localhost:6379"
rchannel = "objbomber"
bfiltername = "objbomber"
hllname = "objbomber"
sleep = None
userhomedir = os.path.expanduser("~")
# flippin' switches...
for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("--dservers"):
dservers = a
elif o in ("--queue"):
dqueue = [a]
elif o in ("--secret"):
secret = a
elif o in ("--bserver"):
bserver = a
elif o in ("--hserver"):
hserver = a
elif o in ("--rserver"):
rserver = a
elif o in ("--rchannel"):
rchannel = a
elif o in ("--bfiltername"):
bfiltername = a
elif o in ("--hllname"):
hllname = a
elif o in ("--mode"):
if a in modes:
mode = a
else:
usage()
sys.exit()
elif o in ("--listen"):
mode_listen = True
elif o in ("--check"):
mode_check = True
elif o in ("--sleep"):
sleep = int(a)
else:
assert False, "unhandled option"
checkdqueue = dqueue + ".check"
if sleep in (None, 0):
sleep = 0.0001
# mode must be set
if not mode:
usage()
sys.exit()
# Handler for the cryptographic signatures
# TODO: secret should be "secret" + a version number
s = Serializer(secret)
# config basics
datadir = userhomedir + "/.objbomber"
# prepare servers and queue lists
dservers = dservers.split(",")
bserver = [bserver]
hserver = hserver
# all modes use Disque
logging.info("Connecting to Disque...")
disque_client = Client(dservers)
disque_client.connect()
if mode in ("check", "listen"):
logging.info("Creating Bloomd Client...")
bloomd_client = BloomdClient(bserver)
bfilter = bloomd_client.create_filter(bfiltername)
# add pyhlld
logging.info("Creating HLLD Client... - not yet used")
hlld_client = HlldClient(hserver)
hll = hlld_client.create_set(hllname)
if mode in ("check", "listen", "generate", "subscriber"):
# add redis hll & pubsub
logging.info("Creating Redis Client...")
rhost, rport = rserver.split(":")
redd = redis.StrictRedis(host=rhost, port=rport, db=0)
redpubsub = redd.pubsub()
if mode in ("subscriber"):
redpubsub.subscribe(rchannel)
if mode in ("generate"):
# TODO: check on how well LevelDB handles
# multiple clients
db = leveldb.LevelDB(datadir + '/notary')
# special mode to handle our first run
# TODO: push into a function
# TODO: handle filesystem errors
# TODO: reconsider using Cement for all of this
# TODO: generate an instance GUID
if mode == "initialize":
UUID = uuid.uuid4()
logging.info("Our system UUID is now: %s" % UUID)
# TODO: save and load this uuid
# check to see if there is a ~/.objbomber directory, quit if there is
# TODO: this does not handle errors in initalization
logging.info("Checking for .objbomber in %s..." % userhomedir)
if os.path.exists(datadir):
logging.info("Already been initialized!")
# TODO: print some information about how to handle this
sys.exit()
# TODO: make one
os.mkdir(datadir, 0700)
# generate our RSA signing key
# TODO: make # of bits in key a flag
logging.info("Begining to create our notary key.")
logging.info("Reading from RNG.")
rng = Random.new().read
logging.info("Generating RSA key...")
privRSAkey = RSA.generate(4096, rng)
privPEM = privRSAkey.exportKey()
pubRSAkey = privRSAkey.publickey()
pubPEM = pubRSAkey.exportKey()
logging.info("Key generated.")
# save privkey to disk
with open(datadir + "/privkey.pem", "w") as keyfile:
keyfile.write(privPEM)
keyfile.close()
os.chmod(datadir + "/privkey.pem", 0700)
logging.info("Unencrypted RSA key written to disk.")
# save the pubkey
with open(datadir + "/pubkey.pem", "w") as keyfile:
keyfile.write(pubPEM)
keyfile.close()
logging.info("Public RSA key written to disk.")
logging.info("Creating crypto notary storage.")
leveldb.LevelDB(datadir + '/notary')
# we absolutely must quit here, or we will get stuck in
# an infinate loop
sys.exit()
# load our secret key (TODO: this is probably better as try/exc)
# and build our contexts
with open(datadir + "/privkey.pem", "r") as keyfile:
privRSAkey = RSA.importKey(keyfile.read())
while True:
# TODO: Adaptive Mode - this mode should peek the queues, and
# make a decision about where this thread can make the most
# impact on its own.
if mode == "adaptive":
# TODO: Do some queue peeking.
# TODO: Make some decisions about which mode to adapt to.
pass
# TODO: All modes should be placed into functions.
# Listen Mode - Listens to the queue, pulls out jobs,
# validates the signature, puts them in bloomd
if mode == "listen":
logging.info("Getting Jobs from Disque.")
jobs = disque_client.get_job([dqueue])
print("Got %d jobs." % len(jobs))
for queue_name, job_id, job in jobs:
logging.debug("Handling a job: %s" % job)
try:
job = s.loads(job)
logging.debug("Job Authenticated: %s" % job)
except:
logging.warning("Job did not pass authentication.")
disque_client.nack_job(job_id)
# add to bloom filter
try:
bfilter.add(job)
except:
logging.warning("Job was not added to bloomd.")
disque_client.nack_job(job_id)
try:
hllResponse = hll.add(job)
except:
logging.warning("Job was not added to hlld.")
disque_client.nack_job(job_id)
# TODO: add redis HLL support
# tell disque that this job has been processed
disque_client.ack_job(job_id)
# sign the check job
job = s.dumps(job)
# throw this message on the check queue
disque_client.add_job(checkdqueue, job)
elif mode == "check":
# TODO
# Check the secondary disque queue for checks
# Ask the bloom filter if they have seen this
logging.info("Getting Jobs from Disque.")
jobs = disque_client.get_job([checkdqueue])
for queue_name, job_id, job in jobs:
logging.debug("Checking: %s" % job)
try:
job = s.loads(job)
except:
disque_client.nack_job(job_id)
# we don't NACK on failed cache hits
if job in bfilter:
logging.info("Confirming: %s" % job)
else:
logging.info("Not found in bloom filter: %s" % job)
disque_client.ack_job(job_id)
elif mode == "generate":
# TODO - where will these messages come from?
# for now they will just be random numbers, but
# really we should make them objects to really be
# testing serialization
msg = [random.randint(1000, 1000000),
random.randint(1000, 1000000)]
# itsdangerous serialization & signing
msg = s.dumps(msg)
# Now that this message is serialized, we can sign it again with a
# public key.
# TODO: incorporate saving the signature into the notary records
msghash = SHA.new(msg)
signer = PKCS1_v1_5.new(privRSAkey)
signature = signer.sign(msghash)
assert signer.verify(msghash, signature)
record = {'message': msg, 'signature': signature}
record = pickle.dumps(record)
# send the job over to Disque
# TODO: add more command line flags for queuing
job_id = disque_client.add_job(dqueue, msg)
logging.debug("Added a job to Disque: %s" % msg)
# publish just the signature on redis pubsub
redd.publish(rchannel, signature)
# TODO: save the publication in the notary
# TODO: do more then just save the signatures
# TODO: add a GUID to the key
key = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f")
db.Put(key, record)
# testing the results of leveldb's store, this is a test, and
# an expensive test
sig2 = db.Get(key)
sig2 = pickle.loads(sig2)['signature']
assert signer.verify(msghash, sig2)
elif mode == "subscriber":
msg = redpubsub.get_message()
# TODO: do something useful, like log
if msg:
print("got a message")
time.sleep(sleep)
config = {
#'host': 'd1-redis-addfix.iqi3ba.ng.0001.apse1.cache.amazonaws.com',
#'host': 'data-prod-redis-002.iqi3ba.0001.apse1.cache.amazonaws.com',
'host': 'localhost',
'port':6379,
'db':0,
}
#for redis connection
r = redis.Redis(**config)
if not r.ping():
sentry_client.captureException(
message = "settings.py: Failed to connect to redis",
extra = {"error" : 'ping failed'})
sys.exit()
## for disque connection
client = Client(["127.0.0.1:7711"])
#client = Client(["10.0.4.232:7711"])
try:
client.connect()
except Exception as disque_err:
sentry_client.captureException(message = "settings.py:Failed to connect to disque",
extra={"error" : disque_err})
sys.exit()
catfight_input = config_parser.get("Queues","catfight_input")
catfight_output = config_parser.get("Queues","catfight_output")
if __name__ == '__main__':
publisher.port = 6381
publisher.channel = 'Script'
parser = argparse.ArgumentParser(description='General Queue.')
parser.add_argument("-r", "--runtime", type=str, required=True, help="Path to the runtime configuration file.")
parser.add_argument("-p", "--pipeline", type=str, required=True, help="Path to the pipeline configuration file.")
parser.add_argument("-i", "--id", type=str, required=True, help="Module ID.")
args = parser.parse_args()
module_name, module_id = args.id.split('_')
runtime = json.load(open(args.runtime, 'r'))
pipeline = json.load(open(args.pipeline, 'r'))
disque_client = Client(runtime["Disque_Default"])
disque_client.connect()
# LOGGING #
publisher.info(module_name + ": started to receive & publish.")
nb = 0
while True:
messages = receive(disque_client, pipeline[module_name]["source-queue"])
if len(messages) > 0:
publisher.debug(module_name + ': Got a message')
for qname, job_id, payload in messages:
send(disque_client, pipeline[module_name]["destination-queues"], payload)
disque_client.ack_job(job_id)
nb += 1
import json
import time
import logging
import pickle
logging.basicConfig(level=logging.DEBUG)
from pydisque.client import Client
c = Client(['localhost:7712', 'localhost:7711'])
c.connect()
while True:
t = time.time()
job = [t, "UPDATE", "Testing Pickled Tweets"]
pjob = pickle.dumps(job)
print "sending job", t
c.add_job("twitpush", pjob, replicate=1)
time.sleep(20)
def ack_job(request):
user = request.user
ip = request.META['REMOTE_ADDR']
zone = request.POST['zone']
jobIds = request.POST.getlist('jobIds', [])
if not user.groups.filter(name__in=['admin', 'dba', 'disque']).exists():
logs(user, ip, 'ack job: %s , zone: %s' % (jobIds, zone),
'permission denied')
return HttpResponse(json.dumps({'errcode': 403}),
content_type=DEFAULT_CONTENT_TYPE)
try:
clusterInfo = ClusterInfo.objects.get(name=zone)
print clusterInfo.addr
except ClusterInfo.DoesNotExist:
logs(user, ip, 'ack job: %s' % jobIds,
'unknown disque zone: %s' % zone)
return HttpResponse(json.dumps({
'errcode': 400,
'msg': 'unknown disque zone:%s' % zone
}),
content_type=DEFAULT_CONTENT_TYPE)
except ClusterInfo.MultipleObjectsReturned:
logs(user, ip, 'ack job: %s' % jobIds,
'multi objects returned for zone: %s' % zone)
return HttpResponse(json.dumps({
'errcode':
400,
'msg':
'multi objects returned for zone:%s' % zone
}),
content_type=DEFAULT_CONTENT_TYPE)
except Exception as e:
print e
logs(user, ip, 'ack job: %s , zone: %s' % (jobIds, zone), str(e))
return HttpResponse(json.dumps({
'errcode': 400,
'msg': str(e)
}),
content_type=DEFAULT_CONTENT_TYPE)
if len(jobIds) == 0:
logs(user, ip, 'ack job: zone-%s' % zone, 'empty jobIds')
return HttpResponse(json.dumps({
'errcode': 400,
'msg': 'empty jobIds'
}),
content_type=DEFAULT_CONTENT_TYPE)
jobIds = map(lambda x: x.encode('utf-8'), jobIds)
print user, zone, jobIds
try:
addr = clusterInfo.addr.split(',')
client = Client(addr)
client.connect()
client.ack_job(*jobIds)
except Exception as e:
print e
logs(user, ip, 'ack job: %s , zone: %s' % (jobIds, zone), str(e))
return HttpResponse(json.dumps({
'errcode': 400,
'msg': str(e)
}),
content_type=DEFAULT_CONTENT_TYPE)
logs(user, ip, 'ack job: %s , zone: %s' % (jobIds, zone), 'success')
return HttpResponse(json.dumps({'errcode': 200}),
content_type=DEFAULT_CONTENT_TYPE)
def main():
'''
Initialize authentication, client connection.
'''
message = 'Delete gists, tweets if a TTL is set.'
socket_help = ('a list containing the host, port numbers to listen to; '
'defaults to localhost:7711 (for disque)')
retry_help = 'queue check frequncy (in seconds); defaults to 8'
parser = ArgumentParser(description=message)
parser.add_argument('-s',
'--sockets',
help=socket_help,
default=['localhost:7711'],
dest='sockets',
metavar=('HOST:PORT'),
nargs='+')
parser.add_argument('-d',
'--debug',
help='enable debugging',
action='store_true',
default=False)
parser.add_argument('-r',
'--retry',
help=retry_help,
default=8,
type=int,
metavar=('DELAY'))
args = vars(parser.parse_args())
if args['debug']:
LOGGER.setLevel(DEBUG)
LOGGER.addHandler(HANDLER)
else:
LOGGER.setLevel(INFO)
LOGGER.addHandler(HANDLER)
# Load the credentials.
tokens = load_credentials()
if None in tokens:
LOGGER.error('[load_credentials] unable to load credentials!')
return
try:
# Connect to the redis-queue.
queue = Client(args['sockets'])
queue.connect()
LOGGER.info('[start-daemon]')
queue_info = json.dumps(queue.info(), indent=4)
LOGGER.debug('[queue-init]\n%s', queue_info)
listen(queue, tokens, args['debug'], args['retry'])
except Exception:
LOGGER.error('[error] unable to connect to the redis-queue (disque)!')
except KeyboardInterrupt:
LOGGER.critical('[stop-daemon]')
return
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import time
from pydisque.client import Client
client = Client(["127.0.0.1:7711"])
client.connect()
nb = 0
while True:
messagedata = time.time()
client.add_job("Global", messagedata, async=True)
nb += 1
#time.sleep(.001)
if nb % 1000 == 0:
print(nb)
if nb % 10000 == 0:
print('Done.')
break
# -*- coding: utf-8 -*-
# pylint: disable=print-statement
from __future__ import unicode_literals
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from pydisque.client import Client
from mico.settings import disque_aws, disque_qcd
import json
from django.shortcuts import render
from asset.utils import logs, deny_resubmit
# Create your views here.
disqueAWS = Client(disque_aws)
disqueAWS.connect()
disqueQCD = Client(disque_qcd)
disqueQCD.connect()
clientEnvMap = {'aws': disqueAWS, 'qcd': disqueQCD}
default_content_type = 'application/json'
@login_required
@deny_resubmit(page_key='disque_ack_job')
def ackjob_index(request):
return render(request, 'disque_ack_job.html')
class TestDisque(unittest.TestCase):
"""TestCase class for pydisque."""
testID = None
def setUp(self):
"""Setup the tests."""
self.client = Client(['localhost:7711'])
self.client.connect()
self.testID = "%d.%d" % (time.time(), random.randint(1000, 1000000))
def test_publish_and_receive(self):
"""Test the most important functions of pydisque."""
t1 = str(time.time())
self.client.add_job("test_q", t1, timeout=100)
jobs = self.client.get_job(['test_q'])
assert len(jobs) == 1
for queue_name, job_id, job in jobs:
assert job == six.b(t1)
self.client.ack_job(job_id)
assert len(self.client.get_job(['test_q'], timeout=100)) == 0
def test_nack(self):
"""Fetch the queue, return a job, check that it's back."""
t1 = str(time.time())
queuename = "test_nack." + self.testID
self.client.add_job(queuename, str(t1), timeout=100)
jobs = self.client.get_job([queuename])
# NACK the first read
assert len(jobs) == 1
for queue_name, job_id, job in jobs:
assert len(jobs) == 1
assert job == six.b(t1)
self.client.nack_job(job_id)
# this time ACK it
jobs = self.client.get_job([queuename])
assert len(jobs) == 1
for queue_name, job_id, job in jobs:
assert job == six.b(t1)
self.client.ack_job(job_id)
assert len(self.client.get_job([queuename], timeout=100)) == 0
def test_qpeek(self):
"""
Test qpeek.
Ran into some problems with an ENQUEUE/DEQUEUE test that
was using qpeek, checking core functionality of qpeek().
"""
queuename = "test_qpeek-%s" % self.testID
job_id = self.client.add_job(queuename, "Peek A Boo")
peeked = self.client.qpeek(queuename, 1)
assert peeked[0][1] == job_id
def test_qscan(self):
"""
Test the qscan function.
This test relies on add_job() being functional, and
the local disque not being a disque proxy to a mesh.
TODO: unique the queues with self.testID.
"""
t1 = str(time.time())
self.client.add_job("q1", t1, timeout=100)
self.client.add_job("q2", t1, timeout=100)
qb = self.client.qscan()
assert qb[0]
assert qb[1]
assert six.b("q1") in qb[1]
assert six.b("q2") in qb[1]
def test_jscan(self):
"""Simple test of the jscan function."""
t1 = time.time()
queuename = "test_jscan-%s" % self.testID
j1 = self.client.add_job(queuename, str(t1), timeout=100)
jerbs = self.client.jscan(queue=queuename)
assert j1 in jerbs[1]
def test_del_job(self):
"""Simple test of del_job, needs qpeek.
FIXME: This function has grown ugly.
"""
t1 = time.time()
queuename = "test_del_job-%s" % self.testID
j1 = self.client.add_job(queuename, str(t1))
jerbs = self.client.qpeek(queuename, 1)
jlist = []
for item in jerbs:
jlist.append(item[1])
assert j1 in jlist
self.client.del_job(j1)
jerbs = self.client.qpeek(queuename, 1)
jlist = []
for item in jerbs:
jlist.append(item[1])
assert j1 not in jerbs
def test_qlen(self):
"""Simple test of qlen."""
queuename = "test_qlen-%s" % self.testID
lengthOfTest = 100
test_job = "Useless Job."
for x in range(lengthOfTest):
self.client.add_job(queuename, test_job)
assert self.client.qlen(queuename) == lengthOfTest
def test_qstat(self):
"""Testing QSTAT (default behavior)."""
queuename = "test_qstat-%s" % self.testID
testqueue = ["a", "b", "c"]
for x in testqueue:
self.client.add_job(queuename, x)
stat = self.client.qstat(queuename)
# check the basics
assert 'jobs-in' in stat
assert 'jobs-out' in stat
def test_qstat_dict(self):
"""Testing QSTAT's (new dict behavior)."""
queuename = "test_qstat_dict-%s" % self.testID
testqueue = ["a", "b", "c"]
for x in testqueue:
self.client.add_job(queuename, x)
stat = self.client.qstat(queuename, True)
assert stat.get('jobs-in', None) is not None
assert stat.get('jobs-out', None) is not None
def test_shownack(self):
"""Test that NACK and SHOW work appropriately."""
queuename = "test_show-%s" % self.testID
test_job = "Show me."
self.client.add_job(queuename, test_job)
jobs = self.client.get_job([queuename])
for queue_name, job_id, job in jobs:
self.client.nack_job(job_id)
shown = self.client.show(job_id, True)
assert shown.get('body') == test_job
assert shown.get('nacks') == 1
def test_pause(self):
"""Test that a PAUSE message is acknowledged."""
queuename = "test_show-%s" % self.testID
test_job = "Jerbs, they are a thing"
self.client.pause(queuename, kw_in=True)
try:
job_id = self.client.add_job(queuename, test_job)
except ResponseError:
pass
# can we add a job again?
self.client.pause(queuename, kw_none=True)
job_id = self.client.add_job(queuename, test_job)
jobs = self.client.get_job([queuename])
# TODO(canardleteer): add a test of PAUSE SHOW
def test_get_job(self):
queue_name = "test_get_job." + self.testID
job = str(time.time())
job_id = self.client.add_job(queue_name, job)
expected = [(queue_name, job_id, job)]
got = self.client.get_job([queue_name], withcounters=False)
assert expected == got
def test_get_job_withcounters(self):
queue_name = "test_get_job." + self.testID
job = str(time.time())
job_id = self.client.add_job(queue_name, job)
nacks = 0
additional_deliveries = 0
expected = [(queue_name, job_id, job, nacks, additional_deliveries)]
got = self.client.get_job([queue_name], withcounters=True)
assert expected == got
class TestDisque(unittest.TestCase):
"""TestCase class for pydisque."""
def setUp(self):
"""Setup the tests."""
self.client = Client(['localhost:7711'])
self.client.connect()
def test_publish_and_receive(self):
"""Test the most important functions of pydisque."""
t1 = str(time.time())
self.client.add_job("test_q", t1, timeout=100)
jobs = self.client.get_job(['test_q'])
assert len(jobs) == 1
for queue_name, job_id, job in jobs:
assert job == six.b(t1)
self.client.ack_job(job_id)
assert len(self.client.get_job(['test_q'], timeout=100)) == 0
def test_nack(self):
"""Fetch the queue, return a job, check that it's back."""
t1 = str(time.time())
self.client.add_job("test_nack_q", str(t1), timeout=100)
jobs = self.client.get_job(['test_nack_q'])
# NACK the first read
assert len(jobs) == 1
for queue_name, job_id, job in jobs:
assert len(jobs) == 1
assert job == six.b(t1)
self.client.nack_job(job_id)
# this time ACK it
jobs = self.client.get_job(['test_nack_q'])
assert len(jobs) == 1
for queue_name, job_id, job in jobs:
assert job == six.b(t1)
self.client.ack_job(job_id)
assert len(self.client.get_job(['test_nack_q'], timeout=100)) == 0
def test_qscan(self):
"""
Test the qscan function.
This test relies on add_job() being functional, and
the local disque not being a disque proxy to a mesh.
"""
t1 = str(time.time())
qa = self.client.qscan()
# print "Cursor: %s Jobs: %s" % (qa[0], qa[1])
self.client.add_job("q1", t1, timeout=100)
self.client.add_job("q2", t1, timeout=100)
qb = self.client.qscan()
# print "Cursor: %s Jobs: %s" % (qb[0], qb[1])
assert qb[0]
assert qb[1]
# i am thinking we need some kind of master 'clear queue'
# command in disque, hopefully not just for the purposes of
# making this unit test more effective...
assert six.b("q1") in qb[1]
assert six.b("q2") in qb[1]
def test_jscan(self):
"""Simple test of the jscan function."""
t1 = time.time()
queuename = "test_jscan-%d" % random.randint(1000, 1000000)
j1 = self.client.add_job(queuename, str(t1), timeout=100)
jerbs = self.client.jscan(queue=queuename)
assert j1 in jerbs[1]
def test_del_job(self):
"""Simple test of del_job, needs jscan."""
t1 = time.time()
queuename = "test_del_job-%d" % random.randint(1000, 1000000)
j1 = self.client.add_job(queuename, str(t1))
jerbs = self.client.jscan(queue=queuename)
assert j1 in jerbs[1]
self.client.del_job(j1)
jerbs = self.client.jscan(queue=queuename)
assert j1 not in jerbs[1]
def test_qlen(self):
"""Simple test of qlen."""
queuename = "test_qlen-%d" % random.randint(1000, 1000000)
lengthOfTest = 100
test_job = "Useless Job."
for x in range(lengthOfTest):
self.client.add_job(queuename, test_job)
assert self.client.qlen(queuename) == lengthOfTest
def main():
'''
Validate arguments; send data to the message bus.
'''
message = 'Push data to the message bus.'
socket_help = ('a list containing the host, port numbers to listen to; '
'defaults to localhost:7711 (for disque)')
ttl_help = ('a TTL (in seconds) for the data on Twitter and GitHub; '
'if not specified, the data will remain forever')
parser = ArgumentParser(description=message)
parser.add_argument('-s',
'--sockets',
help=socket_help,
default=['localhost:7711'],
dest='sockets',
metavar=('HOST:PORT'),
nargs='+')
parser.add_argument('-d',
'--debug',
help='enable debugging',
action='store_true',
default=False)
parser.add_argument('-r',
'--recipient',
help='keybase-id to send',
required=True,
metavar=('KEYBASE-ID'))
parser.add_argument('-t',
'--ttl',
help=ttl_help,
default=0,
type=int,
metavar=('N'))
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-i', '--in-file', metavar=('FILE'), default=None)
group.add_argument('-m', '--message', type=str)
args = vars(parser.parse_args())
if args['debug']:
LOGGER.setLevel(DEBUG)
LOGGER.addHandler(HANDLER)
else:
LOGGER.setLevel(INFO)
LOGGER.addHandler(HANDLER)
plaintext, queue = None, None
if args['in_file']:
name = args['in_file']
if not re.match(r'^text\/.*', magic.from_file(name, mime=True)):
LOGGER.error('[file-error] input-file mimetype should be text/.*')
return
else:
plaintext = open(name, 'r').read()
else:
plaintext = args['message']
try:
# Instantiate a connection to the queue only if a TTL is specified.
if args['ttl']:
queue = Client(args['sockets'])
queue.connect()
queue_info = json.dumps(queue.info(), indent=4)
LOGGER.debug('[queue-init]\n%s', queue_info)
auth = load_credentials()
if None in auth:
LOGGER.error('[load_credentials] unable to load credentials!')
return
send(plaintext=plaintext,
auth=auth,
recipient=args['recipient'],
ttl=args['ttl'],
queue=queue,
debug=args['debug'])
except Exception:
LOGGER.error('[error] unable to connect to the redis-queue (disque)!')
def connect(servers):
global disque
disque = Client(servers)
disque.connect()
