def test_build_timer_cron(self): s1 = app.build_timer(self.app, "* 1 * 2 *", self.tick) self.assertIs(IServiceCollection(self.app), s1.parent) self.assertIsInstance(s1, cron.CrontabTimerService) self.assertEqual("* 1 * 2 *", s1.cronline) s2 = app.build_timer(self.app, ['*', '5-10', '*', '*', '*'], self.tick) self.assertIs(IServiceCollection(self.app), s1.parent) self.assertIsInstance(s1, cron.CrontabTimerService) self.assertEqual("* 5-10 * * *", s2.cronline)
def test_build_interval(self): s1 = app.build_timer(self.app, 1234, self.tick) self.assertIs(IServiceCollection(self.app), s1.parent) self.assertIsInstance(s1, cron.IntervalTimerService) self.assertEqual(1234, s1.interval) s1 = app.build_timer(self.app, " 999 ", self.tick) self.assertIs(IServiceCollection(self.app), s1.parent) self.assertIsInstance(s1, cron.IntervalTimerService) self.assertEqual(999, s1.interval)
def poll(self): app = IServiceCollection(self.app, self.app) launch = app.getServiceNamed('launcher') cur_pro_num = len(launch.processes) # print cur_pro_num, 'fffffffssss' key = "stats:{m}:info".format(m=self.slave_id) redishget_url = '{dispatch_host}/redishset'. \ format(dispatch_host=self.dispatch_host) redishget_data = { "key": key, "field": "cur_pro_num", "value": cur_pro_num } requests.post( redishget_url, redishget_data ) # self.redis_conn.hset(key, cur_pro_num, 'self.node_info') redisexpire_url = '{dispatch_host}/redisexpire'. \ format(dispatch_host=self.dispatch_host) redisexpire_data = {"key": key, "time": 10} requests.post(redisexpire_url, redisexpire_data) #self.redis_conn.expire(key, 10) if self.dq.pending: return for p, q in iteritems(self.queues): c = yield maybeDeferred(q.count) if c: msg = yield maybeDeferred(q.pop) if msg is not None: # In case of a concurrently accessed queue returnValue(self.dq.put(self._message(msg, p)))
def launcher(self): """ Copied from website.Root Should do some refactory to avoid this duplicated code """ app = IServiceCollection(self.app, self.app) return app.getServiceNamed('launcher')
def render_GET(self, request): _sys_services = [] ms = MultiService(self.app) desc = act = '' for srv in IServiceCollection(self.app): if srv.name == "WebAdmin": continue if srv.name == '{0}'.format(cfg['Project']['app_name'] + ' Application'): desc = '''Main web application service. {0}'''.format(cfg['Project']['app_desc']) act = ms.get_start_date() service = { 'name': srv.name, 'running': srv.running, 'description': desc, 'activation': act } _sys_services.append(service) for srv in _sys_services: if srv in ms.get_services().keys(): srv['description'] = ms.get_service(srv).get_description() srv['activation'] = ms.get_service(srv).get_activation() return json.dumps(_sys_services)
def render_POST(self, txrequest): # args = native_stringify_dict(copy(txrequest.args), keys_only=False) # settings = args.pop('setting', []) # settings = dict(x.split('=', 1) for x in settings) # args = dict((k, v[0]) for k, v in args.items()) # project = args.pop('project') # spider = args.pop('spider') # version = args.get('_version', '') # spiders = get_spider_list(project, version=version) # if not spider in spiders: # return {"status": "error", "message": "spider '%s' not found" % spider} # args['settings'] = settings # jobid = args.pop('jobid', uuid.uuid1().hex) # # args['_job'] = jobid # self.root.scheduler.schedule(project, spider, **args) args = native_stringify_dict(copy(txrequest.args), keys_only=False) args = dict((k, v[0]) for k, v in args.items()) crawlid = args.pop('crawlid') appid = args.pop('appid') app = IServiceCollection(self.root.app, self.root.app) self.launcher = app.getServiceNamed('launcher') self.root.launcher._spawn_process(appid, crawlid) return {"node_name": self.root.nodename, "status": "ok"}
def poll(self): app = IServiceCollection(self.app, self.app) launch = app.getServiceNamed('launcher') spiders = launch.processes.values() running = [{ "spider": s.spider, "jobid": s.job, } for s in spiders] # [{'spider': 'link', 'jobid': 'fd967e32f53b11e7a966485b39c53ff1'}, # {'spider': 'link', 'jobid': '03ba6c7ef53c11e7a966485b39c53ff1'}] # print '!!!!!!!!!!!!!!!!!!!', running job_num = len(running) self.socket.send(str(job_num)) # data = self.socket.recv(1024) # print 'data',data # print 'data send to server !!!!!' if self.dq.pending: return for p, q in iteritems(self.queues): c = yield maybeDeferred(q.count) if c: msg = yield maybeDeferred(q.pop) if msg is not None: # In case of a concurrently accessed queue returnValue(self.dq.put(self._message(msg, p)))
def test_deletionDisownsParent(self): """ Test that a deleted C{self.portType} no longer shows up in the children list of the service which used to be its parent. """ port = self.port(store=self.store, portNumber=self.lowPortNumber, factory=self.factory) port.setServiceParent(self.store) port.deleteFromStore() service = IServiceCollection(self.store) self.failIfIn(port, list(service))
def run(self): self.preApplication() self.application = self.createOrGetApplication() self.logger.start(self.application) sc = IServiceCollection(self.application) # reactor is already running, so we just start the service collection sc.startService() return self.application
def test_applicationComponents(self): """ Check L{twisted.application.service.Application} instantiation. """ app = Application("app-name") self.assertTrue(verifyObject(IService, IService(app))) self.assertTrue(verifyObject(IServiceCollection, IServiceCollection(app))) self.assertTrue(verifyObject(IProcess, IProcess(app))) self.assertTrue(verifyObject(IPersistable, IPersistable(app)))
def test_serviceParent(self): """ Test that C{self.portType} is a child of the store service after it is installed. """ port = self.port(store=self.store) installOn(port, self.store) service = IServiceCollection(self.store) self.failUnlessIn(port, list(service))
def cb_twistd(self, *la): if self.running: IServiceCollection(self.app).stopService() self.running = False else: sys.path.insert(0, os.path.abspath(os.getcwd())) sys.argv = TWISTD.split(' ') config = ServerOptions() config.parseOptions() self.app = AndroidApplicationRunner(config).run() self.running = True
def render_POST(self, request): self.serviceName = request.args.get('name', None) for srv in IServiceCollection(self.app): if srv.running and srv.name == self.serviceName[0]: self.srv = srv stop = defer.maybeDeferred(srv.stopService) if stop != None: stop.addCallback(self._success, request) stop.addErrback(self._fail, self.srv, request) return server.NOT_DONE_YET return json.dumps({'success': True, 'name': self.serviceName[0]})
def startService(self): app = IServiceCollection(self.app, self.app) self.poller = app.getServiceNamed('poller') self.fetcher = app.getServiceNamed('fetcher') self.amqp = app.getServiceNamed('amqp') self.task_storage = app.getServiceNamed('task_storage') self.check_storage_path() for slot in range(self.max_proc): self._wait_for_project(slot) log.msg(format='Flowder %(version)s started: max_proc=%(max_proc)r', version=__version__, max_proc=self.max_proc, system='Launcher')
def __init__(self, app, config): resource.Resource.__init__(self) self.app = app self.IApp = IServiceCollection(app, app) self.debug = config.getboolean('debug', False) self.nodename = config.get('node_name', socket.gethostname()) static_serve_path = config.get('static_serve_path', 'files') storage_path = config.get('storage_path') self.putChild('', Home(self)) self.putChild('jobs', Jobs(self)) self.putChild(static_serve_path, File(storage_path)) services = config.items('services', ()) for servName, servClsName in services: servCls = load_object(servClsName) self.putChild(servName, servCls(self))
def __init__(self, app, settings): self.IApp = IServiceCollection(app, app) self.signal_manager = get_signal_manager(app) self.settings = settings self._conn = None self.conn_parameters = None self._client = None self._channel = None self._stopping = None self._consumer_tag = None self._closing = False self._running = False self._publish_meta = None self._message_number_in = 0 self._message_number_out = 0 self._queue_in = None self._queue_out = None self._lock = threading.Lock() self._in_retry = dict() self._cached_messages = [] self.conn_retry_interval = 0 self.app_id = settings.get('app_id', 'fw0') self.consume_interval = settings.getint("consume_interval", 2, section='amqp') self.exchange_name = settings.get("exchange_name", 'flowder-ex', section='amqp') self.queue_in_name = settings.get("queue_in_name", 'flowder-in-queue', section='amqp') self.queue_in_routing_key = settings.get("queue_in_routing_key", 'flowder.in', section='amqp') self.queue_out_name = settings.get("queue_out_name", 'flowder-out-queue', section='amqp') self.queue_out_routing_key = settings.get("queue_out_routing_key", 'flowder.out', section='amqp')
def render_POST(self, request): ms = MultiService(self.app) config = ConfigManager() self.serviceName = request.args.get('name', None) for srv in IServiceCollection(self.app): if not srv.running and srv.name == self.serviceName[0]: start = defer.maybeDeferred(srv.startService) self.srv = srv if start != None: start.addCallback(self._success, request) start.addErrback(self._fail, self.srv, request) return server.NOT_DONE_YET if ms.get_service(self.serviceName[0]) != None: port = ms.get_service(self.serviceName[0]).get_port() else: port = cfg['Project']['app_port'] return json.dumps({ 'success': True, 'name': self.serviceName[0], 'port': port })
def poll(self): # regex = 'dispatch:{slaveId}:*:*:*'.format(slaveId=SlaveID) # for key in self.redis_conn.scan_iter(match=regex): # signal = self.redis_conn.get(key) # userId = key.split(':')[2] # taskId = key.split(':')[3] req = Request('{dispatch_host}/hastask?slaveid={slaveid}'.format( slaveid=self.slave_id, dispatch_host=self.dispatch_host)) res = urllib2.urlopen(req) messages = res.read() messages_json = json.loads(messages) for message in messages_json: signal = message['signal'] userId = message['userId'] taskId = message['taskId'] if signal == 'start': # 启动命令。启动并将该key删除 app = IServiceCollection(self.app, self.app) launcher = app.getServiceNamed('launcher') launcher._spawn_process(userId, taskId) print '启动命令' else: # 暂停命令 url = '{dispatch_host}/stoptask'.format( dispatch_host=self.dispatch_host) pid = signal.split('&')[1] data = { "slaveId": self.slave_id, "userId": userId, "taskId": taskId, "pid": pid } if requests.post(url, data).status_code == 200: print '终止命令' else: print '未正常终止'
def __init__(self, root): self._root = IServiceCollection(root)
def setServiceParent(self, parent): IServiceCollection(parent).addService(self) self.parent = parent
def disownServiceParent(self): IServiceCollection(self.parent).removeService(self) self.parent = None
def setServiceParent(self, parent): if self._parent is not None: self.disownServiceParent() parent = IServiceCollection(parent, parent) self._parent = parent self._parent.addService(self)
def setServiceParent(self, parent): """ Do the standard Axiom thing to make sure this service becomes a child of the top-level store service. """ IServiceCollection(parent).addService(self)
def poller(self): app = IServiceCollection(self.app, self.app) return app.getServiceNamed('poller')
def launcher(self): app = IServiceCollection(self.app, self.app) return app.getServiceNamed('launcher')
from tessw.service.reloadable import Application # ==== # Main # ==== options, cmdline_opts = read_options() startLogging(console=cmdline_opts.console, filepath=cmdline_opts.log_file) # ------------------------------------------------ # Assemble application from its service components # ------------------------------------------------ application = Application("tessw") serviceCollection = IServiceCollection(application) supvrService = SupervisorService(options['global']) supvrService.setName(SUPVR_SERVICE) supvrService.setServiceParent(serviceCollection) mqttService = MQTTService(options['mqtt']) mqttService.setName(MQTT_SERVICE) mqttService.setServiceParent(serviceCollection) # All Photometers under the Supewrvisor Service N = options['global']['nphotom'] for i in range(1, N+1): label = 'phot'+str(i) tesswService = PhotometerService(options[label], label) tesswService.setName(PHOTOMETER_SERVICE + ' ' + str(i))