def poll(self):
app = IServiceCollection(self.app, self.app)
launch = app.getServiceNamed('launcher')
spiders = launch.processes.values()
running = [{
"spider": s.spider,
"jobid": s.job,
} for s in spiders]
# [{'spider': 'link', 'jobid': 'fd967e32f53b11e7a966485b39c53ff1'},
# {'spider': 'link', 'jobid': '03ba6c7ef53c11e7a966485b39c53ff1'}]
# print '!!!!!!!!!!!!!!!!!!!', running
job_num = len(running)
self.socket.send(str(job_num))
# data = self.socket.recv(1024)
# print 'data',data
# print 'data send to server !!!!!'
if self.dq.pending:
return
for p, q in iteritems(self.queues):
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
if msg is not None: # In case of a concurrently accessed queue
returnValue(self.dq.put(self._message(msg, p)))
def launcher(self):
"""
Copied from website.Root
Should do some refactory to avoid this duplicated code
"""
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed('launcher')
def poll(self):
app = IServiceCollection(self.app, self.app)
launch = app.getServiceNamed('launcher')
cur_pro_num = len(launch.processes)
# print cur_pro_num, 'fffffffssss'
key = "stats:{m}:info".format(m=self.slave_id)
redishget_url = '{dispatch_host}/redishset'. \
format(dispatch_host=self.dispatch_host)
redishget_data = {
"key": key,
"field": "cur_pro_num",
"value": cur_pro_num
}
requests.post(
redishget_url, redishget_data
) # self.redis_conn.hset(key, cur_pro_num, 'self.node_info')
redisexpire_url = '{dispatch_host}/redisexpire'. \
format(dispatch_host=self.dispatch_host)
redisexpire_data = {"key": key, "time": 10}
requests.post(redisexpire_url,
redisexpire_data) #self.redis_conn.expire(key, 10)
if self.dq.pending:
return
for p, q in iteritems(self.queues):
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
if msg is not None: # In case of a concurrently accessed queue
returnValue(self.dq.put(self._message(msg, p)))
def launcher(self):
"""
Copied from website.Root
Should do some refactory to avoid this duplicated code
"""
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed('launcher')
class Root(resource.Resource):
def __init__(self, app, config):
resource.Resource.__init__(self)
self.app = app
self.IApp = IServiceCollection(app, app)
self.debug = config.getboolean('debug', False)
self.nodename = config.get('node_name', socket.gethostname())
static_serve_path = config.get('static_serve_path', 'files')
storage_path = config.get('storage_path')
self.putChild('', Home(self))
self.putChild('jobs', Jobs(self))
self.putChild(static_serve_path, File(storage_path))
services = config.items('services', ())
for servName, servClsName in services:
servCls = load_object(servClsName)
self.putChild(servName, servCls(self))
def request_permitted(self, request):
# @TODO log not permitted requests
return request.getClientIP() in self.app.trusted_clients.keys()
@property
def launcher(self):
return self.IApp.getServiceNamed('launcher')
@property
def scheduler(self):
return self.IApp.getServiceNamed('scheduler')
@property
def poller(self):
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed('poller')
def render_POST(self, txrequest):
# args = native_stringify_dict(copy(txrequest.args), keys_only=False)
# settings = args.pop('setting', [])
# settings = dict(x.split('=', 1) for x in settings)
# args = dict((k, v[0]) for k, v in args.items())
# project = args.pop('project')
# spider = args.pop('spider')
# version = args.get('_version', '')
# spiders = get_spider_list(project, version=version)
# if not spider in spiders:
# return {"status": "error", "message": "spider '%s' not found" % spider}
# args['settings'] = settings
# jobid = args.pop('jobid', uuid.uuid1().hex)
#
# args['_job'] = jobid
# self.root.scheduler.schedule(project, spider, **args)
args = native_stringify_dict(copy(txrequest.args), keys_only=False)
args = dict((k, v[0]) for k, v in args.items())
crawlid = args.pop('crawlid')
appid = args.pop('appid')
app = IServiceCollection(self.root.app, self.root.app)
self.launcher = app.getServiceNamed('launcher')
self.root.launcher._spawn_process(appid, crawlid)
return {"node_name": self.root.nodename, "status": "ok"}
def test_build_timer_cron(self):
s1 = app.build_timer(self.app, "* 1 * 2 *", self.tick)
self.assertIs(IServiceCollection(self.app), s1.parent)
self.assertIsInstance(s1, cron.CrontabTimerService)
self.assertEqual("* 1 * 2 *", s1.cronline)
s2 = app.build_timer(self.app, ['*', '5-10', '*', '*', '*'], self.tick)
self.assertIs(IServiceCollection(self.app), s1.parent)
self.assertIsInstance(s1, cron.CrontabTimerService)
self.assertEqual("* 5-10 * * *", s2.cronline)
def run(self):
self.preApplication()
self.application = self.createOrGetApplication()
self.logger.start(self.application)
sc = IServiceCollection(self.application)
# reactor is already running, so we just start the service collection
sc.startService()
return self.application
def run(self):
self.preApplication()
self.application = self.createOrGetApplication()
self.logger.start(self.application)
sc = IServiceCollection(self.application)
# reactor is already running, so we just start the service collection
sc.startService()
return self.application
def test_build_interval(self):
s1 = app.build_timer(self.app, 1234, self.tick)
self.assertIs(IServiceCollection(self.app), s1.parent)
self.assertIsInstance(s1, cron.IntervalTimerService)
self.assertEqual(1234, s1.interval)
s1 = app.build_timer(self.app, " 999 ", self.tick)
self.assertIs(IServiceCollection(self.app), s1.parent)
self.assertIsInstance(s1, cron.IntervalTimerService)
self.assertEqual(999, s1.interval)
def startService(self):
app = IServiceCollection(self.app, self.app)
self.poller = app.getServiceNamed('poller')
self.fetcher = app.getServiceNamed('fetcher')
self.amqp = app.getServiceNamed('amqp')
self.task_storage = app.getServiceNamed('task_storage')
self.check_storage_path()
for slot in range(self.max_proc):
self._wait_for_project(slot)
log.msg(format='Flowder %(version)s started: max_proc=%(max_proc)r',
version=__version__, max_proc=self.max_proc, system='Launcher')
def test_as_app(self):
"""The agent class can be accessed as an application."""
app = BaseAgent().as_app()
multi_service = IService(app, None)
self.assertTrue(IServiceCollection.providedBy(multi_service))
services = list(multi_service)
self.assertEqual(len(services), 1)
def render_GET(self, request):
_sys_services = []
ms = MultiService(self.app)
desc = act = ''
for srv in IServiceCollection(self.app):
if srv.name == "WebAdmin": continue
if srv.name == '{0}'.format(cfg['Project']['app_name'] +
' Application'):
desc = '''Main web application service.
{0}'''.format(cfg['Project']['app_desc'])
act = ms.get_start_date()
service = {
'name': srv.name,
'running': srv.running,
'description': desc,
'activation': act
}
_sys_services.append(service)
for srv in _sys_services:
if srv in ms.get_services().keys():
srv['description'] = ms.get_service(srv).get_description()
srv['activation'] = ms.get_service(srv).get_activation()
return json.dumps(_sys_services)
def test_as_app(self):
"""The agent class can be accessed as an application."""
app = BaseAgent().as_app()
multi_service = IService(app, None)
self.assertTrue(IServiceCollection.providedBy(multi_service))
services = list(multi_service)
self.assertEqual(len(services), 1)
def __init__(self, app, settings):
self.IApp = IServiceCollection(app, app)
self.signal_manager = get_signal_manager(app)
self.settings = settings
self._conn = None
self.conn_parameters = None
self._client = None
self._channel = None
self._stopping = None
self._consumer_tag = None
self._closing = False
self._running = False
self._publish_meta = None
self._message_number_in = 0
self._message_number_out = 0
self._queue_in = None
self._queue_out = None
self._lock = threading.Lock()
self._in_retry = dict()
self._cached_messages = []
self.conn_retry_interval = 0
self.app_id = settings.get('app_id', 'fw0')
self.consume_interval = settings.getint("consume_interval", 2, section='amqp')
self.exchange_name = settings.get("exchange_name", 'flowder-ex', section='amqp')
self.queue_in_name = settings.get("queue_in_name", 'flowder-in-queue', section='amqp')
self.queue_in_routing_key = settings.get("queue_in_routing_key", 'flowder.in', section='amqp')
self.queue_out_name = settings.get("queue_out_name", 'flowder-out-queue', section='amqp')
self.queue_out_routing_key = settings.get("queue_out_routing_key", 'flowder.out', section='amqp')
def __init__(self, app, config):
resource.Resource.__init__(self)
self.app = app
self.IApp = IServiceCollection(app, app)
self.debug = config.getboolean('debug', False)
self.nodename = config.get('node_name', socket.gethostname())
static_serve_path = config.get('static_serve_path', 'files')
storage_path = config.get('storage_path')
self.putChild('', Home(self))
self.putChild('jobs', Jobs(self))
self.putChild(static_serve_path, File(storage_path))
services = config.items('services', ())
for servName, servClsName in services:
servCls = load_object(servClsName)
self.putChild(servName, servCls(self))
def __init__(self, app, settings):
self.IApp = IServiceCollection(app, app)
self.signal_manager = get_signal_manager(app)
self.settings = settings
self._conn = None
self.conn_parameters = None
self._client = None
self._channel = None
self._stopping = None
self._consumer_tag = None
self._closing = False
self._running = False
self._publish_meta = None
self._message_number_in = 0
self._message_number_out = 0
self._queue_in = None
self._queue_out = None
self._lock = threading.Lock()
self._in_retry = dict()
self._cached_messages = []
self.conn_retry_interval = 0
self.app_id = settings.get('app_id', 'fw0')
self.consume_interval = settings.getint("consume_interval",
2,
section='amqp')
self.exchange_name = settings.get("exchange_name",
'flowder-ex',
section='amqp')
self.queue_in_name = settings.get("queue_in_name",
'flowder-in-queue',
section='amqp')
self.queue_in_routing_key = settings.get("queue_in_routing_key",
'flowder.in',
section='amqp')
self.queue_out_name = settings.get("queue_out_name",
'flowder-out-queue',
section='amqp')
self.queue_out_routing_key = settings.get("queue_out_routing_key",
'flowder.out',
section='amqp')
def test_serviceParent(self):
"""
Test that C{self.portType} is a child of the store service after it is
installed.
"""
port = self.port(store=self.store)
installOn(port, self.store)
service = IServiceCollection(self.store)
self.failUnlessIn(port, list(service))
def test_deletionDisownsParent(self):
"""
Test that a deleted C{self.portType} no longer shows up in the children list
of the service which used to be its parent.
"""
port = self.port(store=self.store, portNumber=self.lowPortNumber, factory=self.factory)
port.setServiceParent(self.store)
port.deleteFromStore()
service = IServiceCollection(self.store)
self.failIfIn(port, list(service))
def test_applicationComponents(self):
"""
Check L{twisted.application.service.Application} instantiation.
"""
app = Application("app-name")
self.assertTrue(verifyObject(IService, IService(app)))
self.assertTrue(verifyObject(IServiceCollection, IServiceCollection(app)))
self.assertTrue(verifyObject(IProcess, IProcess(app)))
self.assertTrue(verifyObject(IPersistable, IPersistable(app)))
def inServiceHierarchy(svc, predicate):
"""
Find services in the service collection which satisfy the given predicate.
"""
for subsvc in svc.services:
if IServiceCollection.providedBy(subsvc):
for value in inServiceHierarchy(subsvc, predicate):
yield value
if predicate(subsvc):
yield subsvc
def inServiceHierarchy(svc, predicate):
"""
Find services in the service collection which satisfy the given predicate.
"""
for subsvc in svc.services:
if IServiceCollection.providedBy(subsvc):
for value in inServiceHierarchy(subsvc, predicate):
yield value
if predicate(subsvc):
yield subsvc
def _scan_app_services(acc, root, prefix):
vc = 0
for s in root:
sname = s.name
if not sname:
vc += 1
sname = "$%d" % vc
acc[prefix + sname] = s
if IServiceCollection.providedBy(s):
_scan_app_services(acc, s, sname + ".")
return acc
def cb_twistd(self, *la):
if self.running:
IServiceCollection(self.app).stopService()
self.running = False
else:
sys.path.insert(0, os.path.abspath(os.getcwd()))
sys.argv = TWISTD.split(' ')
config = ServerOptions()
config.parseOptions()
self.app = AndroidApplicationRunner(config).run()
self.running = True
class Services(object):
implements(IServices)
def __init__(self, app):
self.app = app
self.collect = IServiceCollection(self.app)
_multi = []
_services = []
def add(self, name, services):
log.msg(
'bit.core.services: Services.add %s, %s' % (name, services))
if not isinstance(services, dict):
services.setName(name)
services.setServiceParent(self.collect)
self._services.append(name)
return
add = True
if name in self._multi:
plug_services = self.collect.getServiceNamed(name)
add = False
else:
plug_services = MultiService()
plug_services.setName(name)
self._multi.append(name)
for sid, s in services.items():
s.setName(sid)
s.setServiceParent(plug_services)
if add:
plug_services.setServiceParent(self.collect)
@property
def services(self):
_services = {}
for service in (self._services + self._multi):
_services[service] = self.collect.getServiceNamed(service)
return _services
def render_POST(self, request):
self.serviceName = request.args.get('name', None)
for srv in IServiceCollection(self.app):
if srv.running and srv.name == self.serviceName[0]:
self.srv = srv
stop = defer.maybeDeferred(srv.stopService)
if stop != None:
stop.addCallback(self._success, request)
stop.addErrback(self._fail, self.srv, request)
return server.NOT_DONE_YET
return json.dumps({'success': True, 'name': self.serviceName[0]})
def poll(self):
# regex = 'dispatch:{slaveId}:*:*:*'.format(slaveId=SlaveID)
# for key in self.redis_conn.scan_iter(match=regex):
# signal = self.redis_conn.get(key)
# userId = key.split(':')[2]
# taskId = key.split(':')[3]
req = Request('{dispatch_host}/hastask?slaveid={slaveid}'.format(
slaveid=self.slave_id, dispatch_host=self.dispatch_host))
res = urllib2.urlopen(req)
messages = res.read()
messages_json = json.loads(messages)
for message in messages_json:
signal = message['signal']
userId = message['userId']
taskId = message['taskId']
if signal == 'start': # 启动命令。启动并将该key删除
app = IServiceCollection(self.app, self.app)
launcher = app.getServiceNamed('launcher')
launcher._spawn_process(userId, taskId)
print '启动命令'
else: # 暂停命令
url = '{dispatch_host}/stoptask'.format(
dispatch_host=self.dispatch_host)
pid = signal.split('&')[1]
data = {
"slaveId": self.slave_id,
"userId": userId,
"taskId": taskId,
"pid": pid
}
if requests.post(url, data).status_code == 200:
print '终止命令'
else:
print '未正常终止'
def _flatten(self, name, service):
klass = '%s.%s' % (
service.__module__, service.__class__.__name__)
_service = {}
_service['services'] = []
_service['multi'] = False
if IServiceCollection.providedBy(service):
_service['multi'] = True
for sub, subservice in service.namedServices.items():
_service['services'].append(self._flatten(sub, subservice))
for k in ['port', 'running']:
if hasattr(service, k):
_service[k] = str(getattr(service, k))
_service['klass'] = klass
_service['name'] = name
_service['parent'] = service.parent.name
return _service
def _flatten(self, name, agent):
klass = '%s.%s' % (agent.__module__, agent.__class__.__name__)
_agent = {}
_agent['agents'] = []
_agent['multi'] = False
if IServiceCollection.providedBy(agent):
_agent['multi'] = True
for sub, subagent in agent.namedAgents.items():
_agent['agents'].append(
self._flatten(sub, subagent))
for k in ['port', 'running']:
if hasattr(agent, k):
_agent[k] = str(getattr(agent, k))
_agent['klass'] = klass
_agent['name'] = name
_agent['parent'] = '%s.%s' % (
agent.parent.__module__, agent.parent.__class__.__name__)
return _agent
def startService(self):
app = IServiceCollection(self.app, self.app)
self.poller = app.getServiceNamed('poller')
self.fetcher = app.getServiceNamed('fetcher')
self.amqp = app.getServiceNamed('amqp')
self.task_storage = app.getServiceNamed('task_storage')
self.check_storage_path()
for slot in range(self.max_proc):
self._wait_for_project(slot)
log.msg(format='Flowder %(version)s started: max_proc=%(max_proc)r',
version=__version__,
max_proc=self.max_proc,
system='Launcher')
def render_POST(self, request):
ms = MultiService(self.app)
config = ConfigManager()
self.serviceName = request.args.get('name', None)
for srv in IServiceCollection(self.app):
if not srv.running and srv.name == self.serviceName[0]:
start = defer.maybeDeferred(srv.startService)
self.srv = srv
if start != None:
start.addCallback(self._success, request)
start.addErrback(self._fail, self.srv, request)
return server.NOT_DONE_YET
if ms.get_service(self.serviceName[0]) != None:
port = ms.get_service(self.serviceName[0]).get_port()
else:
port = cfg['Project']['app_port']
return json.dumps({
'success': True,
'name': self.serviceName[0],
'port': port
})
def setServiceParent(self, parent):
"""
Do the standard Axiom thing to make sure this service becomes a child
of the top-level store service.
"""
IServiceCollection(parent).addService(self)
def __init__(self, app):
self.app = app
self.collect = IServiceCollection(self.app)
def poller(self):
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed('poller')
from tessw.service.reloadable import Application
# ====
# Main
# ====
options, cmdline_opts = read_options()
startLogging(console=cmdline_opts.console, filepath=cmdline_opts.log_file)
# ------------------------------------------------
# Assemble application from its service components
# ------------------------------------------------
application = Application("tessw")
serviceCollection = IServiceCollection(application)
supvrService = SupervisorService(options['global'])
supvrService.setName(SUPVR_SERVICE)
supvrService.setServiceParent(serviceCollection)
mqttService = MQTTService(options['mqtt'])
mqttService.setName(MQTT_SERVICE)
mqttService.setServiceParent(serviceCollection)
# All Photometers under the Supewrvisor Service
N = options['global']['nphotom']
for i in range(1, N+1):
label = 'phot'+str(i)
tesswService = PhotometerService(options[label], label)
tesswService.setName(PHOTOMETER_SERVICE + ' ' + str(i))
def test_Scheduler_IServiceCollection(self):
self.assertTrue(IServiceCollection.implementedBy(sched.Scheduler),
"Scheduler does not implement IServiceCollection")
class AmqpService(service.Service):
"""
Simple L{service.IService} which consumes an AMQP queue.
:param settings an instance of miutils.config.BaseConfig
:param consume_callback when got an item send the message to this callback
"""
CONSUME_INTERVAL = 10
name = 'amqp'
def __init__(self, app, settings):
self.IApp = IServiceCollection(app, app)
self.signal_manager = get_signal_manager(app)
self.settings = settings
self._conn = None
self.conn_parameters = None
self._client = None
self._channel = None
self._stopping = None
self._consumer_tag = None
self._closing = False
self._running = False
self._publish_meta = None
self._message_number_in = 0
self._message_number_out = 0
self._queue_in = None
self._queue_out = None
self._lock = threading.Lock()
self._in_retry = dict()
self._cached_messages = []
self.conn_retry_interval = 0
self.app_id = settings.get('app_id', 'fw0')
self.consume_interval = settings.getint("consume_interval", 2, section='amqp')
self.exchange_name = settings.get("exchange_name", 'flowder-ex', section='amqp')
self.queue_in_name = settings.get("queue_in_name", 'flowder-in-queue', section='amqp')
self.queue_in_routing_key = settings.get("queue_in_routing_key", 'flowder.in', section='amqp')
self.queue_out_name = settings.get("queue_out_name", 'flowder-out-queue', section='amqp')
self.queue_out_routing_key = settings.get("queue_out_routing_key", 'flowder.out', section='amqp')
@property
def scheduler(self):
return self.IApp.getServiceNamed('scheduler')
def startService(self):
params = {}
if self.settings.get("username", None, section='amqp') \
and self.settings.get("pass", None, section='amqp'):
params['credentials'] = pika_credentials.PlainCredentials(
self.settings.get("username", None, section='amqp'),
self.settings.get("pass", None, section='amqp')
)
if self.settings.getdict("params", dict(), section='amqp'):
params.update(self.settings.getdict("params", dict(), section='amqp'))
if self.settings.get("amqp_vhost", '/'):
params.update({'virtual_host': self.settings.get("vhost", '/', section='amqp')})
parameters = ConnectionParameters(**params)
self._client = protocol.ClientCreator(
reactor,
twisted_connection.TwistedProtocolConnection,
parameters)
self.do_connect()
def do_connect(self):
d = self._client.connectTCP(
self.settings.get("host", "localhost", section='amqp'),
self.settings.getint("port", 5672, section='amqp')
)
d.addCallbacks(lambda protocol: protocol.ready, self.failed)
d.addCallbacks(self.ready, self.failed)
return d
def failed(self, failure):
if failure.check(ChannelClosed):
self.retry_connect()
elif failure.check(ConnectionClosed) or failure.check(ConnectionRefusedError):
self.retry_connect()
else:
log.err("Unhandled failure in Amqp Service....")
failure.printTraceback()
reactor.stop()
@defer.inlineCallbacks
def ready(self, connection):
self._in_retry['connection'] = False
self._in_retry['channel'] = False
if not connection:
raise ConnectionClosed
self._conn = connection
log.msg("AMQP Connection created")
if self._channel:
yield self.check_connection()
self._channel = yield self._conn.channel()
self.conn_retry_interval = 0
log.msg("Setting up exchange and queue")
# Exchange
yield self._channel.exchange_declare(
exchange=self.exchange_name,
type=self.settings.get("exchange_type", 'topic', section='amqp'),
passive=True,
)
# Queue
self._queue_in = yield self._channel.queue_declare(
queue=self.queue_in_name,
auto_delete=False,
exclusive=False,
durable=True,
)
self._queue_out = yield self._channel.queue_declare(
queue=self.queue_out_name,
auto_delete=False,
exclusive=False,
durable=True,
)
# Queue-in > Exchange
yield self._channel.queue_bind(
exchange=self.exchange_name,
queue=self.queue_in_name,
routing_key=self.queue_in_routing_key)
yield self._channel.queue_bind(
exchange=self.exchange_name,
queue=self.queue_out_name,
routing_key=self.queue_out_routing_key)
yield self._channel.basic_qos(prefetch_count=1)
self._running = True
log.msg("Start Consuming %s" % self.queue_in_name)
log.msg("Start Producing %s" % self.queue_out_name)
# Consume queue_in queue
queue_obj, consumer_tag = yield self._channel.basic_consume(queue=self.queue_in_name, no_ack=False)
l = task.LoopingCall(self.read, queue_obj)
l.start(0.01)
@defer.inlineCallbacks
def read(self, queue_obj):
ch, method, properties, msg = yield queue_obj.get()
msg = amqp_message_decode(msg)
log.debug("Consuming msg %s" % msg)
self._message_number_in += 1
self.process_in_message(msg)
time.sleep(self.consume_interval)
yield ch.basic_ack(delivery_tag=method.delivery_tag)
log.debug('Acknowledging message #%s' % self._message_number_in)
@defer.inlineCallbacks
def publish(self, message):
self._message_number_out += 1
amqp_message_update_meta(message, self.get_meta())
amqp_msg = amqp_message_encode(message)
log.debug("Publish message #%s, AMQP message: %s" % (self._message_number_out, amqp_msg))
properties = BasicProperties(
app_id=self.app_id,
content_type='application/json',
content_encoding='utf-8',
delivery_mode=2, # persistent
)
try:
yield self._channel.basic_publish(
self.exchange_name,
self.queue_out_routing_key,
amqp_msg,
properties=properties,
)
except ChannelClosed:
self.retry_channel()
self._cached_messages.append(message)
except AMQPError:
self.retry_connect()
self._cached_messages.append(message)
def retry_connect(self):
with self._lock:
if 'connection' not in self._in_retry or not self._in_retry['connection']:
self.conn_retry_interval += 2
log.err("Connection Closed! retry connecting in %s seconds..." % self.conn_retry_interval)
self._in_retry['connection'] = True
d = task.deferLater(reactor, self.conn_retry_interval, self.do_connect)
d.addErrback(self.failed)
def retry_channel(self):
with self._lock:
if 'channel' not in self._in_retry or not self._in_retry['channel']:
log.err("Channel Closed! retry creating it ...")
self._in_retry['channel'] = True
d = defer.maybeDeferred(self.ready, self._conn)
d.addErrback(self.failed)
def check_connection(self):
d = defer.maybeDeferred(self.get_queue_size)
d.addErrback(self.failed)
@defer.inlineCallbacks
def get_queue_size(self):
queue = yield self._channel.queue_declare(
queue=self.queue_out_name,
passive=True
)
defer.returnValue(queue)
def process_in_message(self, message):
if 'fetch_uri' not in message:
raise InvalidAMQPMessage('Given message has no fetch_uri value.')
message_copy = message.copy()
jobid = uuid.uuid1().hex
_message = dict()
_message['job_id'] = jobid
_message['fetch_uri'] = message_copy.pop('fetch_uri')
# Encode additional fields to send back to client on fetch
try:
_message['settings'] = amqp_message_encode(message_copy)
except:
# @TODO define a more specific exception handling here
raise EncodingError("Can't encode message info. %s" % message)
self.scheduler.schedule(_message)
self.signal_manager.send_catch_log(signal=signals.request_received, jobid=jobid)
def stopService(self):
self._stopping = True
if not self._cached_messages:
return self.really_stop_service(None)
log.warn("Cached messages found, try to publish them before shut down!")
dfds = []
for slot, c in enumerate(self._cached_messages):
dfd = defer.maybeDeferred(self._publish, c, slot)
dfds.append(dfd)
dfd_list = defer.DeferredList(dfds)
dfd_list.addBoth(self.really_stop_service)
return dfd_list
def really_stop_service(self, _):
service.Service.stopService(self)
def get_meta(self):
if not self._publish_meta:
self._publish_meta = {
'app_id': self.app_id,
'timestamp': time.time(),
}
return self._publish_meta
def launcher(self):
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed("launcher")
def disownServiceParent(self):
IServiceCollection(self.parent).removeService(self)
self.parent = None
def setServiceParent(self, parent):
if self._parent is not None:
self.disownServiceParent()
parent = IServiceCollection(parent, parent)
self._parent = parent
self._parent.addService(self)
class AmqpService(service.Service):
"""
Simple L{service.IService} which consumes an AMQP queue.
:param settings an instance of miutils.config.BaseConfig
:param consume_callback when got an item send the message to this callback
"""
CONSUME_INTERVAL = 10
name = 'amqp'
def __init__(self, app, settings):
self.IApp = IServiceCollection(app, app)
self.signal_manager = get_signal_manager(app)
self.settings = settings
self._conn = None
self.conn_parameters = None
self._client = None
self._channel = None
self._stopping = None
self._consumer_tag = None
self._closing = False
self._running = False
self._publish_meta = None
self._message_number_in = 0
self._message_number_out = 0
self._queue_in = None
self._queue_out = None
self._lock = threading.Lock()
self._in_retry = dict()
self._cached_messages = []
self.conn_retry_interval = 0
self.app_id = settings.get('app_id', 'fw0')
self.consume_interval = settings.getint("consume_interval",
2,
section='amqp')
self.exchange_name = settings.get("exchange_name",
'flowder-ex',
section='amqp')
self.queue_in_name = settings.get("queue_in_name",
'flowder-in-queue',
section='amqp')
self.queue_in_routing_key = settings.get("queue_in_routing_key",
'flowder.in',
section='amqp')
self.queue_out_name = settings.get("queue_out_name",
'flowder-out-queue',
section='amqp')
self.queue_out_routing_key = settings.get("queue_out_routing_key",
'flowder.out',
section='amqp')
@property
def scheduler(self):
return self.IApp.getServiceNamed('scheduler')
def startService(self):
params = {}
if self.settings.get("username", None, section='amqp') \
and self.settings.get("pass", None, section='amqp'):
params['credentials'] = pika_credentials.PlainCredentials(
self.settings.get("username", None, section='amqp'),
self.settings.get("pass", None, section='amqp'))
if self.settings.getdict("params", dict(), section='amqp'):
params.update(
self.settings.getdict("params", dict(), section='amqp'))
if self.settings.get("amqp_vhost", '/'):
params.update({
'virtual_host':
self.settings.get("vhost", '/', section='amqp')
})
parameters = ConnectionParameters(**params)
self._client = protocol.ClientCreator(
reactor, twisted_connection.TwistedProtocolConnection, parameters)
self.do_connect()
def do_connect(self):
d = self._client.connectTCP(
self.settings.get("host", "localhost", section='amqp'),
self.settings.getint("port", 5672, section='amqp'))
d.addCallbacks(lambda protocol: protocol.ready, self.failed)
d.addCallbacks(self.ready, self.failed)
return d
def failed(self, failure):
if failure.check(ChannelClosed):
self.retry_connect()
elif failure.check(ConnectionClosed) or failure.check(
ConnectionRefusedError):
self.retry_connect()
else:
log.err("Unhandled failure in Amqp Service....")
failure.printTraceback()
reactor.stop()
@defer.inlineCallbacks
def ready(self, connection):
self._in_retry['connection'] = False
self._in_retry['channel'] = False
if not connection:
raise ConnectionClosed
self._conn = connection
log.msg("AMQP Connection created")
if self._channel:
yield self.check_connection()
self._channel = yield self._conn.channel()
self.conn_retry_interval = 0
log.msg("Setting up exchange and queue")
# Exchange
yield self._channel.exchange_declare(
exchange=self.exchange_name,
type=self.settings.get("exchange_type", 'topic', section='amqp'),
passive=True,
)
# Queue
self._queue_in = yield self._channel.queue_declare(
queue=self.queue_in_name,
auto_delete=False,
exclusive=False,
durable=True,
)
self._queue_out = yield self._channel.queue_declare(
queue=self.queue_out_name,
auto_delete=False,
exclusive=False,
durable=True,
)
# Queue-in > Exchange
yield self._channel.queue_bind(exchange=self.exchange_name,
queue=self.queue_in_name,
routing_key=self.queue_in_routing_key)
yield self._channel.queue_bind(exchange=self.exchange_name,
queue=self.queue_out_name,
routing_key=self.queue_out_routing_key)
yield self._channel.basic_qos(prefetch_count=1)
self._running = True
log.msg("Start Consuming %s" % self.queue_in_name)
log.msg("Start Producing %s" % self.queue_out_name)
# Consume queue_in queue
queue_obj, consumer_tag = yield self._channel.basic_consume(
queue=self.queue_in_name, no_ack=False)
l = task.LoopingCall(self.read, queue_obj)
l.start(0.01)
@defer.inlineCallbacks
def read(self, queue_obj):
ch, method, properties, msg = yield queue_obj.get()
msg = amqp_message_decode(msg)
log.debug("Consuming msg %s" % msg)
self._message_number_in += 1
self.process_in_message(msg)
time.sleep(self.consume_interval)
yield ch.basic_ack(delivery_tag=method.delivery_tag)
log.debug('Acknowledging message #%s' % self._message_number_in)
@defer.inlineCallbacks
def publish(self, message):
self._message_number_out += 1
amqp_message_update_meta(message, self.get_meta())
amqp_msg = amqp_message_encode(message)
log.debug("Publish message #%s, AMQP message: %s" %
(self._message_number_out, amqp_msg))
properties = BasicProperties(
app_id=self.app_id,
content_type='application/json',
content_encoding='utf-8',
delivery_mode=2, # persistent
)
try:
yield self._channel.basic_publish(
self.exchange_name,
self.queue_out_routing_key,
amqp_msg,
properties=properties,
)
except ChannelClosed:
self.retry_channel()
self._cached_messages.append(message)
except AMQPError:
self.retry_connect()
self._cached_messages.append(message)
def retry_connect(self):
with self._lock:
if 'connection' not in self._in_retry or not self._in_retry[
'connection']:
self.conn_retry_interval += 2
log.err(
"Connection Closed! retry connecting in %s seconds..." %
self.conn_retry_interval)
self._in_retry['connection'] = True
d = task.deferLater(reactor, self.conn_retry_interval,
self.do_connect)
d.addErrback(self.failed)
def retry_channel(self):
with self._lock:
if 'channel' not in self._in_retry or not self._in_retry['channel']:
log.err("Channel Closed! retry creating it ...")
self._in_retry['channel'] = True
d = defer.maybeDeferred(self.ready, self._conn)
d.addErrback(self.failed)
def check_connection(self):
d = defer.maybeDeferred(self.get_queue_size)
d.addErrback(self.failed)
@defer.inlineCallbacks
def get_queue_size(self):
queue = yield self._channel.queue_declare(queue=self.queue_out_name,
passive=True)
defer.returnValue(queue)
def process_in_message(self, message):
if 'fetch_uri' not in message:
raise InvalidAMQPMessage('Given message has no fetch_uri value.')
message_copy = message.copy()
jobid = uuid.uuid1().hex
_message = dict()
_message['job_id'] = jobid
_message['fetch_uri'] = message_copy.pop('fetch_uri')
# Encode additional fields to send back to client on fetch
try:
_message['settings'] = amqp_message_encode(message_copy)
except:
# @TODO define a more specific exception handling here
raise EncodingError("Can't encode message info. %s" % message)
self.scheduler.schedule(_message)
self.signal_manager.send_catch_log(signal=signals.request_received,
jobid=jobid)
def stopService(self):
self._stopping = True
if not self._cached_messages:
return self.really_stop_service(None)
log.warn(
"Cached messages found, try to publish them before shut down!")
dfds = []
for slot, c in enumerate(self._cached_messages):
dfd = defer.maybeDeferred(self._publish, c, slot)
dfds.append(dfd)
dfd_list = defer.DeferredList(dfds)
dfd_list.addBoth(self.really_stop_service)
return dfd_list
def really_stop_service(self, _):
service.Service.stopService(self)
def get_meta(self):
if not self._publish_meta:
self._publish_meta = {
'app_id': self.app_id,
'timestamp': time.time(),
}
return self._publish_meta
def launcher(self):
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed('launcher')
def __init__(self, root):
self._root = IServiceCollection(root)
