def test_build_timer_cron(self):
s1 = app.build_timer(self.app, "* 1 * 2 *", self.tick)
self.assertIs(IServiceCollection(self.app), s1.parent)
self.assertIsInstance(s1, cron.CrontabTimerService)
self.assertEqual("* 1 * 2 *", s1.cronline)
s2 = app.build_timer(self.app, ['*', '5-10', '*', '*', '*'], self.tick)
self.assertIs(IServiceCollection(self.app), s1.parent)
self.assertIsInstance(s1, cron.CrontabTimerService)
self.assertEqual("* 5-10 * * *", s2.cronline)
def test_build_interval(self):
s1 = app.build_timer(self.app, 1234, self.tick)
self.assertIs(IServiceCollection(self.app), s1.parent)
self.assertIsInstance(s1, cron.IntervalTimerService)
self.assertEqual(1234, s1.interval)
s1 = app.build_timer(self.app, " 999 ", self.tick)
self.assertIs(IServiceCollection(self.app), s1.parent)
self.assertIsInstance(s1, cron.IntervalTimerService)
self.assertEqual(999, s1.interval)
def poll(self):
app = IServiceCollection(self.app, self.app)
launch = app.getServiceNamed('launcher')
cur_pro_num = len(launch.processes)
# print cur_pro_num, 'fffffffssss'
key = "stats:{m}:info".format(m=self.slave_id)
redishget_url = '{dispatch_host}/redishset'. \
format(dispatch_host=self.dispatch_host)
redishget_data = {
"key": key,
"field": "cur_pro_num",
"value": cur_pro_num
}
requests.post(
redishget_url, redishget_data
) # self.redis_conn.hset(key, cur_pro_num, 'self.node_info')
redisexpire_url = '{dispatch_host}/redisexpire'. \
format(dispatch_host=self.dispatch_host)
redisexpire_data = {"key": key, "time": 10}
requests.post(redisexpire_url,
redisexpire_data) #self.redis_conn.expire(key, 10)
if self.dq.pending:
return
for p, q in iteritems(self.queues):
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
if msg is not None: # In case of a concurrently accessed queue
returnValue(self.dq.put(self._message(msg, p)))
def launcher(self):
"""
Copied from website.Root
Should do some refactory to avoid this duplicated code
"""
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed('launcher')
def render_GET(self, request):
_sys_services = []
ms = MultiService(self.app)
desc = act = ''
for srv in IServiceCollection(self.app):
if srv.name == "WebAdmin": continue
if srv.name == '{0}'.format(cfg['Project']['app_name'] +
' Application'):
desc = '''Main web application service.
{0}'''.format(cfg['Project']['app_desc'])
act = ms.get_start_date()
service = {
'name': srv.name,
'running': srv.running,
'description': desc,
'activation': act
}
_sys_services.append(service)
for srv in _sys_services:
if srv in ms.get_services().keys():
srv['description'] = ms.get_service(srv).get_description()
srv['activation'] = ms.get_service(srv).get_activation()
return json.dumps(_sys_services)
def render_POST(self, txrequest):
# args = native_stringify_dict(copy(txrequest.args), keys_only=False)
# settings = args.pop('setting', [])
# settings = dict(x.split('=', 1) for x in settings)
# args = dict((k, v[0]) for k, v in args.items())
# project = args.pop('project')
# spider = args.pop('spider')
# version = args.get('_version', '')
# spiders = get_spider_list(project, version=version)
# if not spider in spiders:
# return {"status": "error", "message": "spider '%s' not found" % spider}
# args['settings'] = settings
# jobid = args.pop('jobid', uuid.uuid1().hex)
#
# args['_job'] = jobid
# self.root.scheduler.schedule(project, spider, **args)
args = native_stringify_dict(copy(txrequest.args), keys_only=False)
args = dict((k, v[0]) for k, v in args.items())
crawlid = args.pop('crawlid')
appid = args.pop('appid')
app = IServiceCollection(self.root.app, self.root.app)
self.launcher = app.getServiceNamed('launcher')
self.root.launcher._spawn_process(appid, crawlid)
return {"node_name": self.root.nodename, "status": "ok"}
def poll(self):
app = IServiceCollection(self.app, self.app)
launch = app.getServiceNamed('launcher')
spiders = launch.processes.values()
running = [{
"spider": s.spider,
"jobid": s.job,
} for s in spiders]
# [{'spider': 'link', 'jobid': 'fd967e32f53b11e7a966485b39c53ff1'},
# {'spider': 'link', 'jobid': '03ba6c7ef53c11e7a966485b39c53ff1'}]
# print '!!!!!!!!!!!!!!!!!!!', running
job_num = len(running)
self.socket.send(str(job_num))
# data = self.socket.recv(1024)
# print 'data',data
# print 'data send to server !!!!!'
if self.dq.pending:
return
for p, q in iteritems(self.queues):
c = yield maybeDeferred(q.count)
if c:
msg = yield maybeDeferred(q.pop)
if msg is not None: # In case of a concurrently accessed queue
returnValue(self.dq.put(self._message(msg, p)))
def test_deletionDisownsParent(self):
"""
Test that a deleted C{self.portType} no longer shows up in the children list
of the service which used to be its parent.
"""
port = self.port(store=self.store, portNumber=self.lowPortNumber, factory=self.factory)
port.setServiceParent(self.store)
port.deleteFromStore()
service = IServiceCollection(self.store)
self.failIfIn(port, list(service))
def run(self):
self.preApplication()
self.application = self.createOrGetApplication()
self.logger.start(self.application)
sc = IServiceCollection(self.application)
# reactor is already running, so we just start the service collection
sc.startService()
return self.application
def test_applicationComponents(self):
"""
Check L{twisted.application.service.Application} instantiation.
"""
app = Application("app-name")
self.assertTrue(verifyObject(IService, IService(app)))
self.assertTrue(verifyObject(IServiceCollection, IServiceCollection(app)))
self.assertTrue(verifyObject(IProcess, IProcess(app)))
self.assertTrue(verifyObject(IPersistable, IPersistable(app)))
def test_serviceParent(self):
"""
Test that C{self.portType} is a child of the store service after it is
installed.
"""
port = self.port(store=self.store)
installOn(port, self.store)
service = IServiceCollection(self.store)
self.failUnlessIn(port, list(service))
def cb_twistd(self, *la):
if self.running:
IServiceCollection(self.app).stopService()
self.running = False
else:
sys.path.insert(0, os.path.abspath(os.getcwd()))
sys.argv = TWISTD.split(' ')
config = ServerOptions()
config.parseOptions()
self.app = AndroidApplicationRunner(config).run()
self.running = True
def render_POST(self, request):
self.serviceName = request.args.get('name', None)
for srv in IServiceCollection(self.app):
if srv.running and srv.name == self.serviceName[0]:
self.srv = srv
stop = defer.maybeDeferred(srv.stopService)
if stop != None:
stop.addCallback(self._success, request)
stop.addErrback(self._fail, self.srv, request)
return server.NOT_DONE_YET
return json.dumps({'success': True, 'name': self.serviceName[0]})
def startService(self):
app = IServiceCollection(self.app, self.app)
self.poller = app.getServiceNamed('poller')
self.fetcher = app.getServiceNamed('fetcher')
self.amqp = app.getServiceNamed('amqp')
self.task_storage = app.getServiceNamed('task_storage')
self.check_storage_path()
for slot in range(self.max_proc):
self._wait_for_project(slot)
log.msg(format='Flowder %(version)s started: max_proc=%(max_proc)r',
version=__version__,
max_proc=self.max_proc,
system='Launcher')
def __init__(self, app, config):
resource.Resource.__init__(self)
self.app = app
self.IApp = IServiceCollection(app, app)
self.debug = config.getboolean('debug', False)
self.nodename = config.get('node_name', socket.gethostname())
static_serve_path = config.get('static_serve_path', 'files')
storage_path = config.get('storage_path')
self.putChild('', Home(self))
self.putChild('jobs', Jobs(self))
self.putChild(static_serve_path, File(storage_path))
services = config.items('services', ())
for servName, servClsName in services:
servCls = load_object(servClsName)
self.putChild(servName, servCls(self))
def __init__(self, app, settings):
self.IApp = IServiceCollection(app, app)
self.signal_manager = get_signal_manager(app)
self.settings = settings
self._conn = None
self.conn_parameters = None
self._client = None
self._channel = None
self._stopping = None
self._consumer_tag = None
self._closing = False
self._running = False
self._publish_meta = None
self._message_number_in = 0
self._message_number_out = 0
self._queue_in = None
self._queue_out = None
self._lock = threading.Lock()
self._in_retry = dict()
self._cached_messages = []
self.conn_retry_interval = 0
self.app_id = settings.get('app_id', 'fw0')
self.consume_interval = settings.getint("consume_interval",
2,
section='amqp')
self.exchange_name = settings.get("exchange_name",
'flowder-ex',
section='amqp')
self.queue_in_name = settings.get("queue_in_name",
'flowder-in-queue',
section='amqp')
self.queue_in_routing_key = settings.get("queue_in_routing_key",
'flowder.in',
section='amqp')
self.queue_out_name = settings.get("queue_out_name",
'flowder-out-queue',
section='amqp')
self.queue_out_routing_key = settings.get("queue_out_routing_key",
'flowder.out',
section='amqp')
def render_POST(self, request):
ms = MultiService(self.app)
config = ConfigManager()
self.serviceName = request.args.get('name', None)
for srv in IServiceCollection(self.app):
if not srv.running and srv.name == self.serviceName[0]:
start = defer.maybeDeferred(srv.startService)
self.srv = srv
if start != None:
start.addCallback(self._success, request)
start.addErrback(self._fail, self.srv, request)
return server.NOT_DONE_YET
if ms.get_service(self.serviceName[0]) != None:
port = ms.get_service(self.serviceName[0]).get_port()
else:
port = cfg['Project']['app_port']
return json.dumps({
'success': True,
'name': self.serviceName[0],
'port': port
})
def poll(self):
# regex = 'dispatch:{slaveId}:*:*:*'.format(slaveId=SlaveID)
# for key in self.redis_conn.scan_iter(match=regex):
# signal = self.redis_conn.get(key)
# userId = key.split(':')[2]
# taskId = key.split(':')[3]
req = Request('{dispatch_host}/hastask?slaveid={slaveid}'.format(
slaveid=self.slave_id, dispatch_host=self.dispatch_host))
res = urllib2.urlopen(req)
messages = res.read()
messages_json = json.loads(messages)
for message in messages_json:
signal = message['signal']
userId = message['userId']
taskId = message['taskId']
if signal == 'start': # 启动命令。启动并将该key删除
app = IServiceCollection(self.app, self.app)
launcher = app.getServiceNamed('launcher')
launcher._spawn_process(userId, taskId)
print '启动命令'
else: # 暂停命令
url = '{dispatch_host}/stoptask'.format(
dispatch_host=self.dispatch_host)
pid = signal.split('&')[1]
data = {
"slaveId": self.slave_id,
"userId": userId,
"taskId": taskId,
"pid": pid
}
if requests.post(url, data).status_code == 200:
print '终止命令'
else:
print '未正常终止'
def __init__(self, root):
self._root = IServiceCollection(root)
def setServiceParent(self, parent):
IServiceCollection(parent).addService(self)
self.parent = parent
def disownServiceParent(self):
IServiceCollection(self.parent).removeService(self)
self.parent = None
def setServiceParent(self, parent):
if self._parent is not None:
self.disownServiceParent()
parent = IServiceCollection(parent, parent)
self._parent = parent
self._parent.addService(self)
def setServiceParent(self, parent):
"""
Do the standard Axiom thing to make sure this service becomes a child
of the top-level store service.
"""
IServiceCollection(parent).addService(self)
def poller(self):
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed('poller')
def launcher(self):
app = IServiceCollection(self.app, self.app)
return app.getServiceNamed('launcher')
from tessw.service.reloadable import Application
# ====
# Main
# ====
options, cmdline_opts = read_options()
startLogging(console=cmdline_opts.console, filepath=cmdline_opts.log_file)
# ------------------------------------------------
# Assemble application from its service components
# ------------------------------------------------
application = Application("tessw")
serviceCollection = IServiceCollection(application)
supvrService = SupervisorService(options['global'])
supvrService.setName(SUPVR_SERVICE)
supvrService.setServiceParent(serviceCollection)
mqttService = MQTTService(options['mqtt'])
mqttService.setName(MQTT_SERVICE)
mqttService.setServiceParent(serviceCollection)
# All Photometers under the Supewrvisor Service
N = options['global']['nphotom']
for i in range(1, N+1):
label = 'phot'+str(i)
tesswService = PhotometerService(options[label], label)
tesswService.setName(PHOTOMETER_SERVICE + ' ' + str(i))
