def main():
"""Start the poor_consumer."""
try:
opts, args = getopt.getopt(sys.argv[1:], "h:v",
["help", "nack=", "servers=", "queues="])
except getopt.GetoptError as err:
print str(err)
usage()
sys.exit()
# defaults
nack = 0.0
verbose = False
servers = "localhost:7712,localhost:7711"
queues = "test"
for o, a in opts:
if o == "-v":
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("--nack"):
nack = float(a)
elif o in ("--servers"):
servers = a
elif o in ("--queues"):
queues = a
else:
assert False, "unhandled option"
# prepare servers and queus for pydisque
servers = servers.split(",")
queues = queues.split(",")
c = Client(servers)
c.connect()
while True:
jobs = c.get_job(queues)
for queue_name, job_id, job in jobs:
rnd = random.random()
# as this is a test processor, we don't do any validation on
# the actual job body, so lets just pay attention to id's
if rnd >= nack:
print ">>> received job:", job_id
c.ack_job(job_id)
else:
print ">>> bouncing job:", job_id
c.nack_job(job_id)
def main():
'''
Validate arguments; send data to the message bus.
'''
message = 'Push data to the message bus.'
socket_help = ('a list containing the host, port numbers to listen to; '
'defaults to localhost:7711 (for disque)')
ttl_help = ('a TTL (in seconds) for the data on Twitter and GitHub; '
'if not specified, the data will remain forever')
parser = ArgumentParser(description=message)
parser.add_argument('-s',
'--sockets',
help=socket_help,
default=['localhost:7711'],
dest='sockets',
metavar=('HOST:PORT'),
nargs='+')
parser.add_argument('-d',
'--debug',
help='enable debugging',
action='store_true',
default=False)
parser.add_argument('-r',
'--recipient',
help='keybase-id to send',
required=True,
metavar=('KEYBASE-ID'))
parser.add_argument('-t',
'--ttl',
help=ttl_help,
default=0,
type=int,
metavar=('N'))
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('-i', '--in-file', metavar=('FILE'), default=None)
group.add_argument('-m', '--message', type=str)
args = vars(parser.parse_args())
if args['debug']:
LOGGER.setLevel(DEBUG)
LOGGER.addHandler(HANDLER)
else:
LOGGER.setLevel(INFO)
LOGGER.addHandler(HANDLER)
plaintext, queue = None, None
if args['in_file']:
name = args['in_file']
if not re.match(r'^text\/.*', magic.from_file(name, mime=True)):
LOGGER.error('[file-error] input-file mimetype should be text/.*')
return
else:
plaintext = open(name, 'r').read()
else:
plaintext = args['message']
try:
# Instantiate a connection to the queue only if a TTL is specified.
if args['ttl']:
queue = Client(args['sockets'])
queue.connect()
queue_info = json.dumps(queue.info(), indent=4)
LOGGER.debug('[queue-init]\n%s', queue_info)
auth = load_credentials()
if None in auth:
LOGGER.error('[load_credentials] unable to load credentials!')
return
send(plaintext=plaintext,
auth=auth,
recipient=args['recipient'],
ttl=args['ttl'],
queue=queue,
debug=args['debug'])
except Exception:
LOGGER.error('[error] unable to connect to the redis-queue (disque)!')
def setUp(self):
"""Setup the tests."""
self.client = Client(['localhost:7711'])
self.client.connect()
self.testID = "%d.%d" % (time.time(), random.randint(1000, 1000000))
def add_job(request):
user = request.user
ip = request.META['REMOTE_ADDR']
zone = request.POST['zone']
queue = request.POST['queue_name']
# timeout_ms = request.POST['timeout_ms']
# replicate = request.POST['replicate']
# retry_sec = request.POST['retry_sec']
# delay_sec = request.POST['delay_sec']
# ttl_sec = request.POST['ttl_sec']
jobs = request.POST.getlist('jobs', [])
# print user, env, queue, timeout_ms, replicate, retry_sec, delay_sec, ttl_sec
print user, zone, queue
print jobs
if not user.groups.filter(name__in=['admin', 'dba', 'disque']).exists():
logs(user, ip, 'add job: %s - %s' % (zone, queue), 'permission denied')
return HttpResponse(json.dumps({'errcode': 403}),
content_type=DEFAULT_CONTENT_TYPE)
try:
clusterInfo = ClusterInfo.objects.get(name=zone)
print clusterInfo.addr
except ClusterInfo.DoesNotExist:
logs(user, ip, 'add job: %s - %s' % (zone, queue),
'unknown disque zone: %s' % zone)
return HttpResponse(json.dumps({
'errcode': 400,
'msg': 'unknown disque zone:%s' % zone
}),
content_type=DEFAULT_CONTENT_TYPE)
except ClusterInfo.MultipleObjectsReturned:
logs(user, ip, 'add job: %s - %s' % (zone, queue),
'multi objects returned for zone: %s' % zone)
return HttpResponse(json.dumps({
'errcode':
400,
'msg':
'multi objects returned for zone:%s' % zone
}),
content_type=DEFAULT_CONTENT_TYPE)
except Exception as e:
print e
logs(user, ip, 'add job: %s - %s' % (zone, queue), str(e))
return HttpResponse(json.dumps({
'errcode': 400,
'msg': str(e)
}),
content_type=DEFAULT_CONTENT_TYPE)
if (not queue) or len(queue) == 0:
logs(user, ip, 'add job: %s - %s' % (zone, queue), 'empty queue name')
return HttpResponse(json.dumps({
'errcode': 400,
'msg': 'empty queue name'
}),
content_type=DEFAULT_CONTENT_TYPE)
if len(jobs) == 0:
logs(user, ip, 'add job: %s - %s' % (zone, queue), 'empty jobs')
return HttpResponse(json.dumps({
'errcode': 400,
'msg': 'empty jobs'
}),
content_type=DEFAULT_CONTENT_TYPE)
jobs = map(lambda x: x.encode('utf-8'), jobs)
jobIds = []
errJob = []
addr = clusterInfo.addr.split(',')
client = Client(addr)
client.connect()
for job in jobs:
try:
print job
jobId = client.add_job(queue, job)
# jobId = client.add_job(queue, job, timeout=timeout_ms, replicate=replicate, delay=delay_sec, retry=retry_sec, ttl=ttl_sec)
jobIds.append(jobId)
except Exception as e:
print e
errJob.append(job)
logs(user, ip, 'add job: %s - %s' % (zone, queue), 'success')
return HttpResponse(json.dumps({
'errcode': 200,
'jobIds': jobIds,
'failJobs': errJob
}),
content_type=DEFAULT_CONTENT_TYPE)
def ack_job(request):
user = request.user
ip = request.META['REMOTE_ADDR']
zone = request.POST['zone']
jobIds = request.POST.getlist('jobIds', [])
if not user.groups.filter(name__in=['admin', 'dba', 'disque']).exists():
logs(user, ip, 'ack job: %s , zone: %s' % (jobIds, zone),
'permission denied')
return HttpResponse(json.dumps({'errcode': 403}),
content_type=DEFAULT_CONTENT_TYPE)
try:
clusterInfo = ClusterInfo.objects.get(name=zone)
print clusterInfo.addr
except ClusterInfo.DoesNotExist:
logs(user, ip, 'ack job: %s' % jobIds,
'unknown disque zone: %s' % zone)
return HttpResponse(json.dumps({
'errcode': 400,
'msg': 'unknown disque zone:%s' % zone
}),
content_type=DEFAULT_CONTENT_TYPE)
except ClusterInfo.MultipleObjectsReturned:
logs(user, ip, 'ack job: %s' % jobIds,
'multi objects returned for zone: %s' % zone)
return HttpResponse(json.dumps({
'errcode':
400,
'msg':
'multi objects returned for zone:%s' % zone
}),
content_type=DEFAULT_CONTENT_TYPE)
except Exception as e:
print e
logs(user, ip, 'ack job: %s , zone: %s' % (jobIds, zone), str(e))
return HttpResponse(json.dumps({
'errcode': 400,
'msg': str(e)
}),
content_type=DEFAULT_CONTENT_TYPE)
if len(jobIds) == 0:
logs(user, ip, 'ack job: zone-%s' % zone, 'empty jobIds')
return HttpResponse(json.dumps({
'errcode': 400,
'msg': 'empty jobIds'
}),
content_type=DEFAULT_CONTENT_TYPE)
jobIds = map(lambda x: x.encode('utf-8'), jobIds)
print user, zone, jobIds
try:
addr = clusterInfo.addr.split(',')
client = Client(addr)
client.connect()
client.ack_job(*jobIds)
except Exception as e:
print e
logs(user, ip, 'ack job: %s , zone: %s' % (jobIds, zone), str(e))
return HttpResponse(json.dumps({
'errcode': 400,
'msg': str(e)
}),
content_type=DEFAULT_CONTENT_TYPE)
logs(user, ip, 'ack job: %s , zone: %s' % (jobIds, zone), 'success')
return HttpResponse(json.dumps({'errcode': 200}),
content_type=DEFAULT_CONTENT_TYPE)
# -*- coding: utf-8 -*-
# pylint: disable=print-statement
from __future__ import unicode_literals
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from pydisque.client import Client
from mico.settings import disque_aws, disque_qcd
import json
from django.shortcuts import render
from asset.utils import logs, deny_resubmit
# Create your views here.
disqueAWS = Client(disque_aws)
disqueAWS.connect()
disqueQCD = Client(disque_qcd)
disqueQCD.connect()
clientEnvMap = {'aws': disqueAWS, 'qcd': disqueQCD}
default_content_type = 'application/json'
@login_required
@deny_resubmit(page_key='disque_ack_job')
def ackjob_index(request):
return render(request, 'disque_ack_job.html')
def main():
'''
This is the main method, validate args, load credentials, start the daemon.
'''
message = 'Listen to tweets; dump them to the queue.'
socket_help = ('a list containing the host, port numbers to listen to; '
'defaults to localhost:7711 (for disque)')
parser = ArgumentParser(description=message)
parser.add_argument('-s',
'--sockets',
help=socket_help,
default=['localhost:7711'],
dest='sockets',
metavar=('HOST:PORT'),
nargs='+')
parser.add_argument('-c',
'--channels',
help='Twitter accounts to follow',
dest='channels',
metavar=('CHANNEL'),
nargs='+',
required=True)
parser.add_argument('-d',
'--debug',
help='enable debugging',
action='store_true',
default=False)
args = vars(parser.parse_args())
if args['debug']:
LOGGER.setLevel(DEBUG)
LOGGER.addHandler(HANDLER)
else:
LOGGER.setLevel(INFO)
LOGGER.addHandler(HANDLER)
try:
# Connect to the redis-queue.
queue = Client(args['sockets'])
queue.connect()
LOGGER.info('[start-daemon]')
queue_info = json.dumps(queue.info(), indent=4)
LOGGER.debug('[queue-init]\n%s', queue_info)
# Load credentials, initialize authentication module, listen to tweets.
api = load_credentials()
if not api:
LOGGER.error('[load_credentials] unable to load credentials!')
return
listener = StreamDaemon(queue)
streamer = tweepy.Stream(auth=api.auth, listener=listener)
args['channels'] = [re.sub('@', '', _) for _ in args['channels']]
streamer.userstream(track=args['channels'])
except Exception:
LOGGER.error('[error] unknown error')
LOGGER.error('[error] unable to connect to the redis-queue (disque)!')
except KeyboardInterrupt:
LOGGER.critical('[stop-daemon]')
return
def main():
'''
Initialize authentication, client connection.
'''
message = 'Delete gists, tweets if a TTL is set.'
socket_help = ('a list containing the host, port numbers to listen to; '
'defaults to localhost:7711 (for disque)')
retry_help = 'queue check frequncy (in seconds); defaults to 8'
parser = ArgumentParser(description=message)
parser.add_argument('-s',
'--sockets',
help=socket_help,
default=['localhost:7711'],
dest='sockets',
metavar=('HOST:PORT'),
nargs='+')
parser.add_argument('-d',
'--debug',
help='enable debugging',
action='store_true',
default=False)
parser.add_argument('-r',
'--retry',
help=retry_help,
default=8,
type=int,
metavar=('DELAY'))
args = vars(parser.parse_args())
if args['debug']:
LOGGER.setLevel(DEBUG)
LOGGER.addHandler(HANDLER)
else:
LOGGER.setLevel(INFO)
LOGGER.addHandler(HANDLER)
# Load the credentials.
tokens = load_credentials()
if None in tokens:
LOGGER.error('[load_credentials] unable to load credentials!')
return
try:
# Connect to the redis-queue.
queue = Client(args['sockets'])
queue.connect()
LOGGER.info('[start-daemon]')
queue_info = json.dumps(queue.info(), indent=4)
LOGGER.debug('[queue-init]\n%s', queue_info)
listen(queue, tokens, args['debug'], args['retry'])
except Exception:
LOGGER.error('[error] unable to connect to the redis-queue (disque)!')
except KeyboardInterrupt:
LOGGER.critical('[stop-daemon]')
return
import json
import time
import logging
logging.basicConfig(level=logging.DEBUG)
from pydisque.client import Client
c = Client(['localhost:7712', 'localhost:7711'])
c.connect()
while True:
t = time.time()
print "sending job", t
c.add_job("test", json.dumps(["print", "hello", "world", t]), replicate=1, timeout=100)
time.sleep(2)
def __init__(self, conf):
self.conf = conf
self.client = Client([':'.join([self.host, str(self.port)])])
self.client.connect()
config = {
#'host': 'd1-redis-addfix.iqi3ba.ng.0001.apse1.cache.amazonaws.com',
#'host': 'data-prod-redis-002.iqi3ba.0001.apse1.cache.amazonaws.com',
'host': 'localhost',
'port': 6379,
'db': 0,
}
#for redis connection
r = redis.Redis(**config)
if not r.ping():
sentry_client.captureException(
message="settings.py: Failed to connect to redis",
extra={"error": 'ping failed'})
sys.exit()
## for disque connection
client = Client(["127.0.0.1:7711"])
#client = Client(["10.0.4.232:7711"])
try:
client.connect()
except Exception as disque_err:
sentry_client.captureException(
message="settings.py:Failed to connect to disque",
extra={"error": disque_err})
sys.exit()
catfight_input = config_parser.get("Queues", "catfight_input")
catfight_output = config_parser.get("Queues", "catfight_output")
def connect(servers):
global disque
disque = Client(servers)
disque.connect()
