def application(config): app = Application("Scrapyd") http_port = config.getint('http_port', 6800) bind_address = config.get('bind_address', '0.0.0.0') poll_interval = config.getfloat('poll_interval', 5) poller = QueuePoller(config) eggstorage = FilesystemEggStorage(config) scheduler = SpiderScheduler(config) environment = Environment(config) app.setComponent(IPoller, poller) app.setComponent(IEggStorage, eggstorage) app.setComponent(ISpiderScheduler, scheduler) app.setComponent(IEnvironment, environment) laupath = config.get('launcher', 'scrapyd.launcher.Launcher') laucls = load_object(laupath) launcher = laucls(config, app) timer = TimerService(poll_interval, poller.poll) webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address) log.msg( format= "Scrapyd web console available at http://%(bind_address)s:%(http_port)s/", bind_address=bind_address, http_port=http_port) launcher.setServiceParent(app) timer.setServiceParent(app) webservice.setServiceParent(app) return app
def application(config): app = Application("Scrapyd") http_port = config.getint('http_port', 6800) bind_address = config.get('bind_address', '0.0.0.0') poller = QueuePoller(config) eggstorage = FilesystemEggStorage(config) scheduler = SpiderScheduler(config) environment = Environment(config) app.setComponent(IPoller, poller) app.setComponent(IEggStorage, eggstorage) app.setComponent(ISpiderScheduler, scheduler) app.setComponent(IEnvironment, environment) laupath = config.get('launcher', 'scrapyd.launcher.Launcher') laucls = load_object(laupath) launcher = laucls(config, app) timer = TimerService(5, poller.poll) webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address) log.msg("Scrapyd web console available at http://%s:%s/" % (bind_address, http_port)) launcher.setServiceParent(app) timer.setServiceParent(app) webservice.setServiceParent(app) return app
def application(config): app = Application("Scrapyd") http_port = int(environ.get('PORT', config.getint('http_port', 6800))) config.cp.set('scrapyd', 'database_url', environ.get('DATABASE_URL')) poller = Psycopg2QueuePoller(config) eggstorage = FilesystemEggStorage(config) scheduler = Psycopg2SpiderScheduler(config) environment = Environment(config) app.setComponent(IPoller, poller) app.setComponent(IEggStorage, eggstorage) app.setComponent(ISpiderScheduler, scheduler) app.setComponent(IEnvironment, environment) launcher = Launcher(config, app) timer = TimerService(5, poller.poll) webservice = TCPServer(http_port, server.Site(Root(config, app))) log.msg("Scrapyd web console available at http://localhost:%s/ (HEROKU)" % http_port) launcher.setServiceParent(app) timer.setServiceParent(app) webservice.setServiceParent(app) return app
def get_application(config): app = Application('Scrapyd') http_port = config.getint('http_port', 6800) bind_address = config.get('bind_address', '0.0.0.0') poll_interval = config.getfloat('poll_interval', 5) poller = QueuePoller(config) eggstorage = FilesystemEggStorage(config) scheduler = SpiderScheduler(config) environment = Environment(config) app.setComponent(IPoller, poller) app.setComponent(IEggStorage, eggstorage) app.setComponent(ISpiderScheduler, scheduler) app.setComponent(IEnvironment, environment) laupath = config.get('launcher', 'scrapyd_mongodb.launcher.Launcher') laucls = load_object(laupath) launcher = laucls(config, app) timer = TimerService(poll_interval, poller.poll) webservice = TCPServer( http_port, server.Site(Root(config, app)), interface=bind_address) log.msg('http://%(bind_address)s:%(http_port)s/' % {'bind_address':bind_address, 'http_port':http_port}) launcher.setServiceParent(app) timer.setServiceParent(app) webservice.setServiceParent(app) return app
def __init__(self, basedir="."): node.Node.__init__(self, basedir) self.started_timestamp = time.time() self.logSource="Client" self.DEFAULT_ENCODING_PARAMETERS = self.DEFAULT_ENCODING_PARAMETERS.copy() self.init_introducer_client() self.init_stats_provider() self.init_secrets() self.init_storage() self.init_control() self.helper = None if self.get_config("helper", "enabled", False, boolean=True): self.init_helper() self._key_generator = KeyGenerator() key_gen_furl = self.get_config("client", "key_generator.furl", None) if key_gen_furl: self.init_key_gen(key_gen_furl) self.init_client() # ControlServer and Helper are attached after Tub startup self.init_ftp_server() self.init_sftp_server() hotline_file = os.path.join(self.basedir, self.SUICIDE_PREVENTION_HOTLINE_FILE) if os.path.exists(hotline_file): age = time.time() - os.stat(hotline_file)[stat.ST_MTIME] self.log("hotline file noticed (%ds old), starting timer" % age) hotline = TimerService(1.0, self._check_hotline, hotline_file) hotline.setServiceParent(self) # this needs to happen last, so it can use getServiceNamed() to # acquire references to StorageServer and other web-statusable things webport = self.get_config("node", "web.port", None) if webport: self.init_web(webport) # strports string
def application(config): app = Application("Scrapyd") http_port = config.getint('http_port', 6800) bind_address = config.get('bind_address', '0.0.0.0') poller = QueuePoller(config) eggstorage = FilesystemEggStorage(config) scheduler = SpiderScheduler(config) environment = Environment(config) app.setComponent(IPoller, poller) app.setComponent(IEggStorage, eggstorage) app.setComponent(ISpiderScheduler, scheduler) app.setComponent(IEnvironment, environment) launcher = Launcher(config, app) timer = TimerService(5, poller.poll) webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address) log.msg("Scrapyd web console available at http://%s:%s/" % (bind_address, http_port)) launcher.setServiceParent(app) timer.setServiceParent(app) webservice.setServiceParent(app) return app
def startService(self): Service.startService(self) self._registered = False self._timer_service = TimerService( self.check_interval.total_seconds(), self._check_certs) self._timer_service.clock = self._clock self._timer_service.startService()
def application(config): app = Application("Scrapyd") http_port = config.getint('http_port', 6800) portal = Portal(PublicHTMLRealm(config, app), [FilePasswordDB(str(config.get('passwd', '')))]) credentialFactory = DigestCredentialFactory("md5", "Go away") poller = QueuePoller(config) eggstorage = FilesystemEggStorage(config) scheduler = SpiderScheduler(config) environment = Environment(config) app.setComponent(IPoller, poller) app.setComponent(IEggStorage, eggstorage) app.setComponent(ISpiderScheduler, scheduler) app.setComponent(IEnvironment, environment) launcher = Launcher(config, app) timer = TimerService(5, poller.poll) webservice = TCPServer(http_port, server.Site(HTTPAuthSessionWrapper(portal, [credentialFactory]))) log.msg("Scrapyd web console available at http://localhost:%s/" % http_port) launcher.setServiceParent(app) timer.setServiceParent(app) webservice.setServiceParent(app) return app
def setUp(self): """ Set up a timer service to test. """ self.timer = TimerService(2, self.call) self.clock = self.timer.clock = task.Clock() self.deferred = Deferred()
def __init__(self, clock=None, enable_monitoring=True): # Order is very important here. First we set the clock to the passed-in # reactor, so that unit tests can fake out the clock if necessary. # Then we call super(). The superclass will set up the structures # required to add parents to this service, which allows the remainder # of this method to succeed. (And do so without the side-effect of # executing calls that shouldn't be executed based on the desired # reactor.) self.clock = clock super().__init__() self.enable_monitoring = enable_monitoring # The last successfully recorded interfaces. self._recorded = None self._monitored = frozenset() self._monitoring_state = {} self._monitoring_mdns = False self._locked = False # Use a named filesystem lock to prevent more than one monitoring # service running on each host machine. This service attempts to # acquire this lock on each loop, and then it holds the lock until the # service stops. self._lock = NetworksMonitoringLock() # Set up child service to update interface. self.interface_monitor = TimerService(self.interval, self.updateInterfaces) self.interface_monitor.setName("updateInterfaces") self.interface_monitor.clock = self.clock self.interface_monitor.setServiceParent(self) self.beaconing_protocol = None
def setup_selfheal_service(clock, config, dispatcher, health_checker, log): """ Setup selfheal timer service and return it. :param clock: :obj:`IReactorTime` provider :param dict config: Configuration dict containing selfheal info :param dispatcher: Effect dispatcher :param health_checker: ``HealthChecker`` object where SelfHeal's health check will be added :param log: :obj:`BoundLog` logger used by service :return: selfheal service or None if relevant config is not found :rtype: :obj:`IService` """ if "selfheal" not in config: return None interval = get_in(["selfheal", "interval"], config, no_default=True) selfheal = SelfHeal(clock, dispatcher, config_value, interval, log) func, lock = zk.locked_logged_func(dispatcher, "/selfheallock", log, "selfheal-lock-acquired", selfheal.setup) health_checker.checks["selfheal"] = zk.create_health_check(lock) sh_timer = TimerService(interval, func) sh_timer.clock = clock return sh_timer
class ContextModalityTimer(object): """Provide a timer service for physical and virtual context modality.""" def __init__(self, contextmodalityid, period): """Initialize a context modality timer.""" self.contextmodalityid = contextmodalityid self.period = period self.timer = None def start(self, callback, *args, **kwargs): """Start the timer with a given period.""" if args: if kwargs: self.timer = TimerService(self.period, callback, args, kwargs) else: self.timer = TimerService(self.period, callback, args) else: if kwargs: self.timer = TimerService(self.period, callback, kwargs) else: self.timer = TimerService(self.period, callback) self.timer.startService() return def stop(self): """'Stop the timer.""" return self.timer.stopService()
def setup_selfheal_service(clock, config, dispatcher, health_checker, log): """ Setup selfheal timer service and return it. :param clock: :obj:`IReactorTime` provider :param dict config: Configuration dict containing selfheal info :param dispatcher: Effect dispatcher :param health_checker: ``HealthChecker`` object where SelfHeal's health check will be added :param log: :obj:`BoundLog` logger used by service :return: selfheal service or None if relevant config is not found :rtype: :obj:`IService` """ if "selfheal" not in config: return None interval = get_in(["selfheal", "interval"], config, no_default=True) selfheal = SelfHeal(clock, dispatcher, config_value, interval, log) func, lock = zk.locked_logged_func( dispatcher, "/selfheallock", log, "selfheal-lock-acquired", selfheal.setup) health_checker.checks["selfheal"] = zk.create_health_check(lock) sh_timer = TimerService(interval, func) sh_timer.clock = clock return sh_timer
def stopService(self): """ Stop this service. This will release buckets partitions it holds """ TimerService.stopService(self) if self.kz_partition.acquired: return self.kz_partition.finish()
def application(config, components=interfaces): app = Application("Scrapyd") http_port = config.getint('http_port', 6800) bind_address = config.get('bind_address', '0.0.0.0') for interface, key in interfaces: path = config.get(key) cls = load_object(path) component = cls(config) app.setComponent(interface, component) poller = component laupath = config.get('launcher', 'scrapyd.launcher.Launcher') laucls = load_object(laupath) launcher = laucls(config, app) poll_every = config.getint("poll_every", 5) timer = TimerService(poll_every, poller.poll) webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address) log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/", bind_address=bind_address, http_port=http_port) launcher.setServiceParent(app) timer.setServiceParent(app) webservice.setServiceParent(app) return app
def __init__(self, kz_client, interval, partitioner_path, buckets, time_boundary, log, got_buckets, clock=None): """ :param log: a bound log :param kz_client: txKazoo client :param partitioner_path: ZooKeeper path, used for partitioning :param buckets: iterable of buckets to distribute between nodes. Ideally there should be at least as many elements as nodes taking part in this partitioner. This should be a sequence of str. :param time_boundary: time to wait for partitioning to stabilize. :param got_buckets: Callable which will be called with a list of buckets when buckets have been allocated to this node. :param clock: clock to use for checking the buckets on an interval. """ MultiService.__init__(self) self.kz_client = kz_client self.partitioner_path = partitioner_path self.buckets = buckets self.log = log self.got_buckets = got_buckets self.time_boundary = time_boundary ts = TimerService(interval, self.check_partition) ts.setServiceParent(self) ts.clock = clock self._old_buckets = []
def application(config): app = Application("Scrapyd") http_port = config.getint('http_port', 6800) poller = QueuePoller(config) eggstorage = FilesystemEggStorage(config) scheduler = SpiderScheduler(config) environment = Environment(config) app.setComponent(IPoller, poller) app.setComponent(IEggStorage, eggstorage) app.setComponent(ISpiderScheduler, scheduler) app.setComponent(IEnvironment, environment) launcher = Launcher(config, app) timer = TimerService(5, poller.poll) root = Root(config, app) root = configRoot(root, config) webservice = TCPServer(http_port, server.Site(root)) log.msg("Scrapyd web console available at http://localhost:%s/" % http_port) launcher.setServiceParent(app) timer.setServiceParent(app) webservice.setServiceParent(app) return app
def __init__(self, reactor, config, log, clock=None, collect=None): """ Initialize the service by connecting to Cassandra and setting up authenticator :param reactor: Twisted reactor for connection purposes :param dict config: All the config necessary to run the service. Comes from config file :param IReactorTime clock: Optional reactor for timer purpose """ self._client = connect_cass_servers(reactor, config['cassandra']) self.log = log self.reactor = reactor self._divergent_groups = {} self.divergent_timeout = get_in( ['metrics', 'divergent_timeout'], config, 3600) self._service = TimerService( get_in(['metrics', 'interval'], config, default=60), collect or self.collect, reactor, config, self.log, client=self._client, authenticator=generate_authenticator(reactor, config['identity'])) self._service.clock = clock or reactor
def startService(self): self.log.info("Starting SQS poller {name} for URI {uri} (every {i}s)", name=self.name, uri=self.uri, i=self.pollinterval) self.timerService = TimerService(self.pollinterval, self.poll) self.timerService.setServiceParent(self) return super().startService()
def cb_start(result): """ Called when the client is ready for operation. """ self._timer_service = TimerService( self.check_interval.total_seconds(), self._check_certs) self._timer_service.clock = self._clock self._timer_service.startService()
def startService(self): """ Start this service. This will start buckets partitioning """ self.kz_partition = self.kz_client.SetPartitioner( self.zk_partition_path, set=set(self.buckets), time_boundary=self.time_boundary) TimerService.startService(self)
def __init__(self, reactor): MultiService.__init__(self) self._deployment_state = DeploymentState() timer = TimerService(1, self._wipe_expired) timer.clock = reactor timer.setServiceParent(self) self._information_wipers = pmap() self._clock = reactor
def __init__(self, server_process): assert isinstance(server_process, ServerProcess), \ repr(server_process) self.__server_process = server_process self.__restore_timer = \ TimerService(WEB_RESTORE_PERIOD.total_seconds(), exceptions_logged(logger)(callInThread), self.__poll_restore_requests_in_thread)
def run(self): self.m_connectorServer.begin(self.m_port) self.m_timer = TimerService(1, self.timer) self.m_timer.startService() from twisted.internet import reactor self.m_isRunning = True reactor.run()
def __init__(self, basedir): service.MultiService.__init__(self) self.basedir = basedir self.clients = {} self.nicknames = {} self.timer = TimerService(self.poll_interval, self.poll) self.timer.setServiceParent(self)
def __init__(self, poller=None, step=300): """ """ if poller is None: raise RuntimeError('poller cannot be None') if not issubclass(type(poller), IPoll): raise TypeError('poller privided is not a subclass of IPoll') self._poller = poller TimerService.__init__(self, step=step, callable=self._poller.poll)
def run(self): self.m_config.connect(self.configCallBack) self.m_timer = TimerService(1, self.timer) self.m_timer.startService() #要放在最后一步 from twisted.internet import reactor self.m_isRunning = True logging.info("reactor run") reactor.run()
def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider, introducer_clients, storage_farm_broker): """ Use :func:`allmydata.client.create_client` to instantiate one of these. """ node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider) self._magic_folders = dict() self.started_timestamp = time.time() self.logSource = "Client" self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy() self.introducer_clients = introducer_clients self.storage_broker = storage_farm_broker self.init_stats_provider() self.init_secrets() self.init_node_key() self.init_storage() self.init_control() self._key_generator = KeyGenerator() key_gen_furl = config.get_config("client", "key_generator.furl", None) if key_gen_furl: log.msg("[client]key_generator.furl= is now ignored, see #2783") self.init_client() self.load_static_servers() self.helper = None if config.get_config("helper", "enabled", False, boolean=True): if not self._is_tub_listening(): raise ValueError("config error: helper is enabled, but tub " "is not listening ('tub.port=' is empty)") self.init_helper() self.init_ftp_server() self.init_sftp_server() self.init_magic_folder() # If the node sees an exit_trigger file, it will poll every second to see # whether the file still exists, and what its mtime is. If the file does not # exist or has not been modified for a given timeout, the node will exit. exit_trigger_file = config.get_config_path(self.EXIT_TRIGGER_FILE) if os.path.exists(exit_trigger_file): age = time.time() - os.stat(exit_trigger_file)[stat.ST_MTIME] self.log("%s file noticed (%ds old), starting timer" % (self.EXIT_TRIGGER_FILE, age)) exit_trigger = TimerService(1.0, self._check_exit_trigger, exit_trigger_file) exit_trigger.setServiceParent(self) # this needs to happen last, so it can use getServiceNamed() to # acquire references to StorageServer and other web-statusable things webport = config.get_config("node", "web.port", None) if webport: self.init_web(webport) # strports string
def __init__(self, poller=None, step=60*5): """ The magic here is calling the TimerService constructor (old style class) to set the polling interval and specify the polling function. """ if poller is None: raise RuntimeError('poller cannot be None') if not issubclass(type(poller), IPoll): raise TypeError('poller provided is not a subclass of IPoll'); self._poller = poller TimerService.__init__(self, step=step, callable=self._poller.poll)
def __init__(self, all_stations, base_config, base_dir, poll_length, file_pattern, send_command, command_helper, complete_command, failed_command): TimerService.__init__(self, poll_length, self.run_poll) self.log = log self.all_stations = all_stations self.base_config = dict(base_config) self.base_folder = base_dir self.file_pattern = file_pattern self.send_command = send_command self.complete_command = complete_command self.failed_command = failed_command self.command_helper = command_helper
class StatsGatherer(Referenceable, service.MultiService): implements(RIStatsGatherer) poll_interval = 60 def __init__(self, basedir): service.MultiService.__init__(self) self.basedir = basedir self.clients = {} self.nicknames = {} self.timer = TimerService(self.poll_interval, self.poll) self.timer.setServiceParent(self) def get_tubid(self, rref): return rref.getRemoteTubID() def remote_provide(self, provider, nickname): tubid = self.get_tubid(provider) if tubid == '<unauth>': print "WARNING: failed to get tubid for %s (%s)" % (provider, nickname) # don't add to clients to poll (polluting data) don't care about disconnect return self.clients[tubid] = provider self.nicknames[tubid] = nickname def poll(self): for tubid, client in self.clients.items(): nickname = self.nicknames.get(tubid) d = client.callRemote('get_stats') d.addCallbacks( self.got_stats, self.lost_client, callbackArgs=(tubid, nickname), errbackArgs=(tubid, )) d.addErrback(self.log_client_error, tubid) def lost_client(self, f, tubid): # this is called lazily, when a get_stats request fails del self.clients[tubid] del self.nicknames[tubid] f.trap(DeadReferenceError) def log_client_error(self, f, tubid): log.msg( "StatsGatherer: error in get_stats(), peerid=%s" % tubid, level=log.UNUSUAL, failure=f) def got_stats(self, stats, tubid, nickname): raise NotImplementedError()
def __init__(self, reactor, interval, k8s, namespace, router): TimerService.__init__( self, interval, divert_errors_to_log(self._check_once, u"router-update"), k8s, namespace, ) # This attribute controls the the reactor used by TimerService to set # up the LoopingCall. self.clock = reactor self._router = router
def __init__(self, server_process): assert isinstance(server_process, ServerProcess), \ repr(server_process) self.server_process = server_process self.lock = RLock() self.__schedule_check_timer = \ TimerService(BACKUP_SCHEDULES_CHECK_PERIOD.total_seconds(), self.__on_schedule_check_timer) self.reread_cache()
def __init__(self, fsnotify_manager): """Constructor. @type fsnotify_manager: AbstractFSNotifyManager """ self.__inner_fsnotify_manager = fsnotify_manager self.__watched_dir_to_cb = {} # mapping from path to callback self.__implicitly_watched_dirs = set() self.__poll_sync_dir_timer = \ TimerService(POLL_SYNC_DIR_FOR_SYMLINKS_PERIOD.total_seconds(), lambda: callInThread( self.__on_poll_sync_dir_timer)) self.__links_map_pair = LinksMapPair(links={}, reverse_links={})
class StatsGatherer(Referenceable, service.MultiService): implements(RIStatsGatherer) poll_interval = 60 def __init__(self, basedir): service.MultiService.__init__(self) self.basedir = basedir self.clients = {} self.nicknames = {} self.timer = TimerService(self.poll_interval, self.poll) self.timer.setServiceParent(self) def get_tubid(self, rref): return rref.getRemoteTubID() def remote_provide(self, provider, nickname): tubid = self.get_tubid(provider) if tubid == '<unauth>': print "WARNING: failed to get tubid for %s (%s)" % (provider, nickname) # don't add to clients to poll (polluting data) don't care about disconnect return self.clients[tubid] = provider self.nicknames[tubid] = nickname def poll(self): for tubid, client in self.clients.items(): nickname = self.nicknames.get(tubid) d = client.callRemote('get_stats') d.addCallbacks(self.got_stats, self.lost_client, callbackArgs=(tubid, nickname), errbackArgs=(tubid, )) d.addErrback(self.log_client_error, tubid) def lost_client(self, f, tubid): # this is called lazily, when a get_stats request fails del self.clients[tubid] del self.nicknames[tubid] f.trap(DeadReferenceError) def log_client_error(self, f, tubid): log.msg("StatsGatherer: error in get_stats(), peerid=%s" % tubid, level=log.UNUSUAL, failure=f) def got_stats(self, stats, tubid, nickname): raise NotImplementedError()
def __init__(self, agent, url, event_listeners, interval=DEFAULT_INTERVAL, state_store=None, TimerService=TimerService, coiterate=coiterate): """ :param agent: a :class:`twisted.web.client.Agent` to use to poll :param url: the url to poll :param event_listeners: listeners that handle a particular event :type event_listeners: `iterable` of `callables` that take an event as an argument :param interval: how often to poll, given in seconds - defaults to 10 :type interval: ``int`` or ``float`` :param state_store: where to store the current polling state :type state_store: :class:`otter.indexer.state.IStateStore` provider :param TimerService: factory (not instance) that produces something like a :class:`twisted.application.internet.TimerService` - defaults to :class:`twisted.application.internet.TimerService` (this parameter is mainly used for dependency injection for testing) :type TimerService: ``callable`` :param coiterate: function that is used to coiterate tasks - defaults to :func:`twisted.internet.task.coiterate` - (this parameter is mainly used for dependency injection for testing) :type coiterate: ``callable`` """ self._url = url self._interval = interval self._timer_service = TimerService(interval, self._do_poll) self._next_url = None self._agent = agent self._state_store = state_store or DummyStateStore() self._event_listeners = event_listeners self._poll_timer = timer('FeedPollerService.poll.{0}'.format(url)) self._fetch_timer = timer('FeedPollerService.fetch.{0}'.format(url)) self._coiterate = coiterate
def __init__(self, basedir="."): node.Node.__init__(self, basedir) # All tub.registerReference must happen *after* we upcall, since # that's what does tub.setLocation() configutil.validate_config(self.config_fname, self.config, _valid_config_sections()) self._magic_folder = None self.started_timestamp = time.time() self.logSource = "Client" self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy() self.init_introducer_clients() self.init_stats_provider() self.init_secrets() self.init_node_key() self.init_storage() self.init_control() self._key_generator = KeyGenerator() key_gen_furl = self.get_config("client", "key_generator.furl", None) if key_gen_furl: log.msg("[client]key_generator.furl= is now ignored, see #2783") self.init_client() self.load_static_servers() self.helper = None if self.get_config("helper", "enabled", False, boolean=True): if not self._tub_is_listening: raise ValueError("config error: helper is enabled, but tub " "is not listening ('tub.port=' is empty)") self.init_helper() self.init_ftp_server() self.init_sftp_server() self.init_magic_folder() # If the node sees an exit_trigger file, it will poll every second to see # whether the file still exists, and what its mtime is. If the file does not # exist or has not been modified for a given timeout, the node will exit. exit_trigger_file = os.path.join(self.basedir, self.EXIT_TRIGGER_FILE) if os.path.exists(exit_trigger_file): age = time.time() - os.stat(exit_trigger_file)[stat.ST_MTIME] self.log("%s file noticed (%ds old), starting timer" % (self.EXIT_TRIGGER_FILE, age)) exit_trigger = TimerService(1.0, self._check_exit_trigger, exit_trigger_file) exit_trigger.setServiceParent(self) # this needs to happen last, so it can use getServiceNamed() to # acquire references to StorageServer and other web-statusable things webport = self.get_config("node", "web.port", None) if webport: self.init_web(webport) # strports string
def __init__(self, is_direction_incoming): """ @param is_direction_incoming: whether the measured direction is incoming (otherwise outgoing). @type is_direction_incoming: bool """ self.__direction_incoming = is_direction_incoming self.__total_bytes = 0 self.__ticks = deque() self.__ticks_lock = Lock() self.__start_time = None self.__timer_service = TimerService( TRAFFIC_NOTIFICATION_PERIOD.total_seconds(), self.__on_timer) self.__started = False
def __init__(self, batchsize, interval, slv_client, clock=None): """ Initializes the scheduler service with batch size and interval :param int batchsize: number of events to fetch on each iteration :param int interval: time between each iteration :param slv_client: a :class:`silverberg.client.CQLClient` or :class:`silverberg.cluster.RoundRobinCassandraCluster` instance used to get lock :param clock: An instance of IReactorTime provider that defaults to reactor if not provided """ from otter.models.cass import LOCK_TABLE_NAME self.lock = BasicLock(slv_client, LOCK_TABLE_NAME, 'schedule', max_retry=0) TimerService.__init__(self, interval, self.check_for_events, batchsize) self.clock = clock self.log = otter_log.bind(system='otter.scheduler')
def test_pickleTimerServiceNotPickleLoopFinished(self): """ When pickling L{internet.TimerService}, it won't pickle L{internet.TimerService._loopFinished}. """ # We need a pickleable callable to test pickling TimerService. So we # can't use self.timer timer = TimerService(1, fakeTargetFunction) timer.startService() dumpedTimer = pickle.dumps(timer) timer.stopService() loadedTimer = pickle.loads(dumpedTimer) nothing = object() value = getattr(loadedTimer, "_loopFinished", nothing) self.assertIdentical(nothing, value)
def application(config): app = Application("Scrapyd") http_port = config.getint('http_port', 6800) bind_address = config.get('bind_address', '127.0.0.1') poll_interval = config.getfloat('poll_interval', 5) poller = QueuePoller(config) eggstorage = FilesystemEggStorage(config) scheduler = SpiderScheduler(config) environment = Environment(config) app.setComponent(IPoller, poller) app.setComponent(IEggStorage, eggstorage) app.setComponent(ISpiderScheduler, scheduler) app.setComponent(IEnvironment, environment) laupath = config.get('launcher', 'scrapyd.launcher.Launcher') laucls = load_object(laupath) launcher = laucls(config, app) timer = TimerService(poll_interval, poller.poll) webpath = config.get('webroot', 'scrapyd.website.Root') webcls = load_object(webpath) username = config.get('username', '') password = config.get('password', '') if username and password: if ':' in username: sys.exit("The `username` option contains illegal character ':', " "check and update the configuration file of Scrapyd") portal = Portal(PublicHTMLRealm(webcls(config, app)), [StringCredentialsChecker(username, password)]) credential_factory = BasicCredentialFactory("Auth") resource = HTTPAuthSessionWrapper(portal, [credential_factory]) log.msg("Basic authentication enabled") else: resource = webcls(config, app) log.msg("Basic authentication disabled as either `username` or `password` is unset") webservice = TCPServer(http_port, server.Site(resource), interface=bind_address) log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/", bind_address=bind_address, http_port=http_port) launcher.setServiceParent(app) timer.setServiceParent(app) webservice.setServiceParent(app) return app
def __init__(self, basedir="."): node.Node.__init__(self, basedir) # All tub.registerReference must happen *after* we upcall, since # that's what does tub.setLocation() configutil.validate_config(self.config_fname, self.config, _valid_config_sections()) self._magic_folder = None self.started_timestamp = time.time() self.logSource="Client" self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy() self.init_introducer_clients() self.init_stats_provider() self.init_secrets() self.init_node_key() self.init_storage() self.init_control() self._key_generator = KeyGenerator() key_gen_furl = self.get_config("client", "key_generator.furl", None) if key_gen_furl: log.msg("[client]key_generator.furl= is now ignored, see #2783") self.init_client() self.load_static_servers() self.helper = None if self.get_config("helper", "enabled", False, boolean=True): if not self._tub_is_listening: raise ValueError("config error: helper is enabled, but tub " "is not listening ('tub.port=' is empty)") self.init_helper() self.init_ftp_server() self.init_sftp_server() self.init_magic_folder() # If the node sees an exit_trigger file, it will poll every second to see # whether the file still exists, and what its mtime is. If the file does not # exist or has not been modified for a given timeout, the node will exit. exit_trigger_file = os.path.join(self.basedir, self.EXIT_TRIGGER_FILE) if os.path.exists(exit_trigger_file): age = time.time() - os.stat(exit_trigger_file)[stat.ST_MTIME] self.log("%s file noticed (%ds old), starting timer" % (self.EXIT_TRIGGER_FILE, age)) exit_trigger = TimerService(1.0, self._check_exit_trigger, exit_trigger_file) exit_trigger.setServiceParent(self) # this needs to happen last, so it can use getServiceNamed() to # acquire references to StorageServer and other web-statusable things webport = self.get_config("node", "web.port", None) if webport: self.init_web(webport) # strports string
def makeService(config, channel_db="relay.sqlite", reactor=reactor): increase_rlimits() parent = MultiService() channel_db = create_or_upgrade_channel_db(config["channel-db"]) usage_db = create_or_upgrade_usage_db(config["usage-db"]) log_file = (os.fdopen(int(config["log-fd"]), "w") if config["log-fd"] is not None else None) server = make_server(channel_db, allow_list=config["allow-list"], advertise_version=config["advertise-version"], signal_error=config["signal-error"], blur_usage=config["blur-usage"], usage_db=usage_db, log_file=log_file, ) server.setServiceParent(parent) rebooted = time.time() def expire(): now = time.time() old = now - CHANNEL_EXPIRATION_TIME server.prune_all_apps(now, old) server.dump_stats(now, rebooted=rebooted) TimerService(EXPIRATION_CHECK_PERIOD, expire).setServiceParent(parent) log_requests = config["blur-usage"] is None site = make_web_server(server, log_requests, config["websocket-protocol-options"]) ep = endpoints.serverFromString(reactor, config["port"]) # to listen StreamServerEndpointService(ep, site).setServiceParent(parent) log.msg("websocket listening on ws://HOSTNAME:PORT/v1") return parent
def __init__(self, jobs): """ jobs : dict {job name : configuration options} """ super().__init__() # For each job configuration, instantiate the requisite components # and string everything together using (multi)service(s). for name, cfg in jobs.items(): self.log.debug("configuring job {name}", name=name) lw = Workspace( sqlite.Engine(":memory:"), fs.LocalDirectory(cfg["directory"], filters=cfg["filters"]), ) # DEBUG rw = Workspace( sqlite.Engine(":memory:"), fs.LocalDirectory("/tmp/wspace", filters=cfg["filters"]), ) # END DEBUG merger = TwoWayMerger(lw, rw) trigger = TimerService(cfg.pop("frequency", .025), merger.sync) self.addService(Job(name, merger, trigger))
def stopService(self): if self.log_file is None: return defer.succeed(None) else: self.log_file.close() self.log_file = None return TimerService.stopService(self)
def __init__(self, *args, **kwargs): """Constructor.""" super(UHostApp, self).__init__(*args, **kwargs) # The manager that just tracks the FS __internal_fsnotify_manager = FSNotifyManager() # The manager that supports the symlinks on toplevel. self.__fsnotify_manager = \ SyncDirFSNotifyProxy(fsnotify_manager=__internal_fsnotify_manager) self.__syncer_starter = None self.auto_start_sync = True on_non_empty_cooling_down_to_store = \ lambda: logger_status_cooling_down_to_store.info( 'Some files cooled down to store recently', extra={'status': True}) on_empty_cooling_down_to_store = \ lambda: logger_status_cooling_down_to_store.info( 'No more files cooling down to store', extra={'status': False}) self.__cooling_down_to_store = \ EventfulDict(_on_non_empty_cb=on_non_empty_cooling_down_to_store, _on_empty_cb=on_empty_cooling_down_to_store) self.__cooling_down_to_store_lock = RLock() self.__file_states_ready_to_write = {} self.__file_states_ready_to_write_lock = Lock() self.__ignore_file_paths = dict() self.__ignore_file_paths_lock = RLock() self.ux = ux.UX() ux_handlers = [ ux.RestoreFSM(self.ux.handle_event), ux.BackupFSM(self.ux.handle_event), ux.SyncFSM(self.ux.handle_event), ux.IdleFSM(self.ux.handle_event), ux.NetworkConnectionFSM(self.ux.handle_event), ux.OccupationHandler(self.ux.handle_event), ux.UXEventForwarder() ] for ux_handler in ux_handlers: self.ux.add_handler(ux_handler) self.__network_connection_is_working = True logger.debug('Will attempt to create a dataset once every %s', CREATE_DATASET_PERIOD) logger.debug('Will publish connection state every %s', PUBLISH_HOST_STATE_PERIOD) self.__all_timer_services = [ # We should not create a dataset until the previous attempt # of dataset creation has been completed. # Thus DelayServiceForCallback rather than TimerService. DelayServiceForCallback(period=CREATE_DATASET_PERIOD, callback=self.__on_create_dataset_timer), TimerService(PUBLISH_HOST_STATE_PERIOD.total_seconds(), exceptions_logged(logger)(self.__publish_host_state)) ]
def execute(self): log.trace("Executing watcher.") self._lastPrintedId = None self._lastError = None if self.config["interval"]: svc = TimerService(self.config["interval"], self.showTimeline) svc.setServiceParent(self) # Since this runs ~forever, just return a Deferred that doesn't call # back. A swift SIGINT will kill it. d = Deferred() else: # Print it once and exit d = self.showTimeline() return d
def application(settings): app = Application("Scraperd") poller = QueuePoller(settings) environment = Environment(settings) app.setComponent(IPoller, poller) app.setComponent(IEnvironment, environment) laupath = settings.get('DAEMON_LAUNCHER', 'scraper.daemon.launcher.Launcher') laucls = load_object(laupath) launcher = laucls(settings, app) poll_interval = settings.getfloat('DAEMON_POLL_INTERVAL', 5) timer = TimerService(poll_interval, poller.poll) launcher.setServiceParent(app) timer.setServiceParent(app) return app