def __init__(self, exchange, response_queue, request_routing_key, max_retries=3, timeout_ms=5000):
self.exchange = exchange
self.response_queue = response_queue
self.request_routing_key = request_routing_key
self.messagebus = MessageBus()
self.max_retries = max_retries # maximum number of times to retry on RPC timeout. -1 indicates infinite retries
self.timeout_ms = timeout_ms # Set to -1 for no timeout
self.polling_interval = 0.01 # in seconds
self.sync_sequence_number = 0
self.messagebus.exchange_declare(exchange, 'direct')
#self.messagebus.queue_delete(queue=response_queue) # clear it in case there are backed up messages (EDIT: it *should* autodelete)
self.messagebus.queue_declare(queue=response_queue, auto_delete=True)
self.messagebus.queue_purge(response_queue)
self.messagebus.queue_bind(response_queue, exchange, routing_key=response_queue)
logger.debug("Response queue '%s' is bound to key '%s' on exchange '%s'" % (response_queue, response_queue, exchange))
def __init__(self,
amqp_channel=None, # If None, a new connection & channel will be created.
exchange=DEFAULT_RPC_EXCHANGE,
request_routing_key=None, # Required
reply_queue=None, # Required
timeout_ms=10000,
max_retries=3):
self.amqp_channel = amqp_channel or MessageBus().channel
self.exchange = exchange
# Required parameters:
for param in ('reply_queue', 'request_routing_key'):
if locals()[param]:
setattr(self, param, locals()[param])
else:
raise AmqpService.ParameterMissing("%s is a required parameter." % param)
self.rpc_channel = RpcChannel(self.exchange, self.reply_queue, self.request_routing_key, max_retries=max_retries, timeout_ms=timeout_ms)
self.amqp_rpc_controller = AmqpRpcController()
class FakeJobCommand(object):
current_footprints = []
messagebus = MessageBus()
messagebus.exchange_declare('ngt.platefile.index', 'direct')
@classmethod
def get_plate_info(klass, somevar):
return (12345, 67890)
@classmethod
def postprocess_job(klass, job, state):
if job.uuid in klass.current_footprints:
del klass.current_footprints[job.uuid]
if state == 'failed':
transaction_id, platefile_id = klass.get_plate_info(job.output)
if transaction_id and platefile_id:
idx_transaction_failed = {
'platefile_id': platefile_id,
'transaction_id': transaction_id
}
request = {
'sequence_number':
0,
'requestor':
'',
'method':
'IndexTransactionFailed',
'payload':
protocols.pack(protobuf.IndexTransactionFailed,
idx_transaction_failed),
}
msg = Message(
protocols.pack(protobuf.BroxtonRequestWrapper, request))
klass.messagebus.basic_publish(msg,
exchange='ngt.platefile.index',
routing_key='index')
logger.debug("Message published: " + str(msg))
return job
from ngt.messaging import messagebus
from ngt.messaging.messagebus import MessageBus
from amqplib.client_0_8 import Message
logger = logging.getLogger('dispatch')
logger.setLevel(logging.INFO)
d_logger = logging.getLogger('dispatch_debug')
#d_logger.addHandler(logging.FileHandler('dispatch.log', 'w') )
#d_logger.setLevel(logging.DEBUG)
#logging.getLogger().setLevel(logging.DEBUG)
#logging.getLogger('protocol').setLevel(logging.DEBUG)
REAPER_SWEEP_INTERVAL = 5 * 60 # Time in seconds between reaper garbage collection sweeps
REAPER_SWEEP_MAX_TIMEOUTS = 1 # Number of times to try pinging a reaper upon sweep before giving up.
mb = MessageBus()
sys.path.insert(0, '../..')
from django.core.management import setup_environ
from ngt import settings
setup_environ(settings)
from models import Reaper
from ngt.jobs.models import Job, JobSet
from django import db
from django.db.models import Q
from commands import jobcommands
command_map = {
'registerReaper': 'register_reaper',
'unregisterReaper': 'unregister_reaper',
from django.http import HttpResponse
from django.shortcuts import render_to_response
from ngt.messaging.messagebus import MessageBus
from ngt.dispatch.forms import JobForm
mbus = MessageBus()
def index(request):
return HttpResponse('Hi from the Master Control Program.')
def initiate_job(command, args = []):
msg = ' '.join([command] + args)
return mbus.basic_publish(msg, routing_key='command')
def test_view(request):
s = initiate_job('ls -l')
return HttpResponse(str(s))
def jobber(request):
form = JobForm()
if request.META['REQUEST_METHOD'] == 'GET':
return render_to_response('dispatch/jobform.html', {'form':form})
elif request.META['REQUEST_METHOD'] == 'POST':
#start the job
initiate_job(request.POST['command'], request.POST['params'].split(' '))
return render_to_response('dispatch/jobform.html', {'form':form, 'message':'Job enqueued.'})
class MosaicJobCommand(JobCommand):
commandname = 'mosaic'
number_of_args = 2
current_footprints = {}
messagebus = MessageBus()
messagebus.exchange_declare('ngt.platefile.index', 'direct')
def check_readiness(self):
if self.job.assets.all()[0].footprint:
footprint = self.job.assets.all()[0].footprint.prepared
for other_footprint in self.current_footprints.values():
if other_footprint.touches(footprint):
return False
else:
return True
else:
return True
def preprocess(self):
if self.job.assets.all()[0].footprint:
self.current_footprints[
self.job.uuid] = self.job.assets.all()[0].footprint.prepared
return self.job
def _get_plate_info(self, output):
m = re.search('Transaction ID: (\d+)', output)
if m:
transaction_id = int(m.groups()[0])
else:
transaction_id = None
m = re.search('Platefile ID: (\d+)', output)
if m:
platefile_id = int(m.groups()[0])
else:
platefile_id = None
return transaction_id, platefile_id
def postprocess(self):
if self.job.uuid in self.current_footprints:
del self.current_footprints[self.job.uuid]
if self.job.status == 'failed':
transaction_id, platefile_id = self._get_plate_info(
self.job.output)
if transaction_id and platefile_id:
idx_transaction_failed = {
'platefile_id': platefile_id,
'transaction_id': transaction_id
}
request = {
'sequence_number':
0,
'requestor':
'',
'method':
'TransactionFailed',
'payload':
protocols.pack(protobuf.IndexTransactionFailed,
idx_transaction_failed),
}
msg = Message(
protocols.pack(protobuf.BroxtonRequestWrapper, request))
self.messagebus.basic_publish(msg,
exchange='ngt.platefile.index_0',
routing_key='index')
return self.job
