def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg("Scrapyd web console available at http://%s:%s/" % (bind_address, http_port))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = int(environ.get('PORT', config.getint('http_port', 6800)))
config.cp.set('scrapyd', 'database_url', environ.get('DATABASE_URL'))
poller = Psycopg2QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = Psycopg2SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
log.msg("Scrapyd web console available at http://localhost:%s/ (HEROKU)"
% http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def get_application(config):
app = Application('Scrapyd')
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd_mongodb.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(
http_port, server.Site(Root(config, app)),
interface=bind_address)
log.msg('http://%(bind_address)s:%(http_port)s/' % {'bind_address':bind_address, 'http_port':http_port})
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def __init__(self, basedir="."):
node.Node.__init__(self, basedir)
self.started_timestamp = time.time()
self.logSource="Client"
self.DEFAULT_ENCODING_PARAMETERS = self.DEFAULT_ENCODING_PARAMETERS.copy()
self.init_introducer_client()
self.init_stats_provider()
self.init_secrets()
self.init_storage()
self.init_control()
self.helper = None
if self.get_config("helper", "enabled", False, boolean=True):
self.init_helper()
self._key_generator = KeyGenerator()
key_gen_furl = self.get_config("client", "key_generator.furl", None)
if key_gen_furl:
self.init_key_gen(key_gen_furl)
self.init_client()
# ControlServer and Helper are attached after Tub startup
self.init_ftp_server()
self.init_sftp_server()
hotline_file = os.path.join(self.basedir,
self.SUICIDE_PREVENTION_HOTLINE_FILE)
if os.path.exists(hotline_file):
age = time.time() - os.stat(hotline_file)[stat.ST_MTIME]
self.log("hotline file noticed (%ds old), starting timer" % age)
hotline = TimerService(1.0, self._check_hotline, hotline_file)
hotline.setServiceParent(self)
# this needs to happen last, so it can use getServiceNamed() to
# acquire references to StorageServer and other web-statusable things
webport = self.get_config("node", "web.port", None)
if webport:
self.init_web(webport) # strports string
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg("Scrapyd web console available at http://%s:%s/" %
(bind_address, http_port))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def startService(self):
Service.startService(self)
self._registered = False
self._timer_service = TimerService(
self.check_interval.total_seconds(), self._check_certs)
self._timer_service.clock = self._clock
self._timer_service.startService()
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
portal = Portal(PublicHTMLRealm(config, app), [FilePasswordDB(str(config.get('passwd', '')))])
credentialFactory = DigestCredentialFactory("md5", "Go away")
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(HTTPAuthSessionWrapper(portal, [credentialFactory])))
log.msg("Scrapyd web console available at http://localhost:%s/" % http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = int(environ.get('PORT', config.getint('http_port', 6800)))
config.cp.set('scrapyd', 'database_url', environ.get('DATABASE_URL'))
poller = Psycopg2QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = Psycopg2SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
log.msg("Scrapyd web console available at http://localhost:%s/ (HEROKU)" %
http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def setUp(self):
"""
Set up a timer service to test.
"""
self.timer = TimerService(2, self.call)
self.clock = self.timer.clock = task.Clock()
self.deferred = Deferred()
def __init__(self, clock=None, enable_monitoring=True):
# Order is very important here. First we set the clock to the passed-in
# reactor, so that unit tests can fake out the clock if necessary.
# Then we call super(). The superclass will set up the structures
# required to add parents to this service, which allows the remainder
# of this method to succeed. (And do so without the side-effect of
# executing calls that shouldn't be executed based on the desired
# reactor.)
self.clock = clock
super().__init__()
self.enable_monitoring = enable_monitoring
# The last successfully recorded interfaces.
self._recorded = None
self._monitored = frozenset()
self._monitoring_state = {}
self._monitoring_mdns = False
self._locked = False
# Use a named filesystem lock to prevent more than one monitoring
# service running on each host machine. This service attempts to
# acquire this lock on each loop, and then it holds the lock until the
# service stops.
self._lock = NetworksMonitoringLock()
# Set up child service to update interface.
self.interface_monitor = TimerService(self.interval,
self.updateInterfaces)
self.interface_monitor.setName("updateInterfaces")
self.interface_monitor.clock = self.clock
self.interface_monitor.setServiceParent(self)
self.beaconing_protocol = None
def setup_selfheal_service(clock, config, dispatcher, health_checker, log):
"""
Setup selfheal timer service and return it.
:param clock: :obj:`IReactorTime` provider
:param dict config: Configuration dict containing selfheal info
:param dispatcher: Effect dispatcher
:param health_checker: ``HealthChecker`` object where SelfHeal's health
check will be added
:param log: :obj:`BoundLog` logger used by service
:return: selfheal service or None if relevant config is not found
:rtype: :obj:`IService`
"""
if "selfheal" not in config:
return None
interval = get_in(["selfheal", "interval"], config, no_default=True)
selfheal = SelfHeal(clock, dispatcher, config_value, interval, log)
func, lock = zk.locked_logged_func(dispatcher, "/selfheallock", log,
"selfheal-lock-acquired",
selfheal.setup)
health_checker.checks["selfheal"] = zk.create_health_check(lock)
sh_timer = TimerService(interval, func)
sh_timer.clock = clock
return sh_timer
class ContextModalityTimer(object):
"""Provide a timer service for physical and virtual context modality."""
def __init__(self, contextmodalityid, period):
"""Initialize a context modality timer."""
self.contextmodalityid = contextmodalityid
self.period = period
self.timer = None
def start(self, callback, *args, **kwargs):
"""Start the timer with a given period."""
if args:
if kwargs:
self.timer = TimerService(self.period, callback, args, kwargs)
else:
self.timer = TimerService(self.period, callback, args)
else:
if kwargs:
self.timer = TimerService(self.period, callback, kwargs)
else:
self.timer = TimerService(self.period, callback)
self.timer.startService()
return
def stop(self):
"""'Stop the timer."""
return self.timer.stopService()
def setup_selfheal_service(clock, config, dispatcher, health_checker, log):
"""
Setup selfheal timer service and return it.
:param clock: :obj:`IReactorTime` provider
:param dict config: Configuration dict containing selfheal info
:param dispatcher: Effect dispatcher
:param health_checker: ``HealthChecker`` object where SelfHeal's health
check will be added
:param log: :obj:`BoundLog` logger used by service
:return: selfheal service or None if relevant config is not found
:rtype: :obj:`IService`
"""
if "selfheal" not in config:
return None
interval = get_in(["selfheal", "interval"], config, no_default=True)
selfheal = SelfHeal(clock, dispatcher, config_value, interval, log)
func, lock = zk.locked_logged_func(
dispatcher, "/selfheallock", log, "selfheal-lock-acquired",
selfheal.setup)
health_checker.checks["selfheal"] = zk.create_health_check(lock)
sh_timer = TimerService(interval, func)
sh_timer.clock = clock
return sh_timer
def stopService(self):
"""
Stop this service. This will release buckets partitions it holds
"""
TimerService.stopService(self)
if self.kz_partition.acquired:
return self.kz_partition.finish()
def application(config, components=interfaces):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
for interface, key in interfaces:
path = config.get(key)
cls = load_object(path)
component = cls(config)
app.setComponent(interface, component)
poller = component
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
poll_every = config.getint("poll_every", 5)
timer = TimerService(poll_every, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def __init__(self,
kz_client,
interval,
partitioner_path,
buckets,
time_boundary,
log,
got_buckets,
clock=None):
"""
:param log: a bound log
:param kz_client: txKazoo client
:param partitioner_path: ZooKeeper path, used for partitioning
:param buckets: iterable of buckets to distribute between
nodes. Ideally there should be at least as many elements as nodes
taking part in this partitioner. This should be a sequence of str.
:param time_boundary: time to wait for partitioning to stabilize.
:param got_buckets: Callable which will be called with a list of
buckets when buckets have been allocated to this node.
:param clock: clock to use for checking the buckets on an interval.
"""
MultiService.__init__(self)
self.kz_client = kz_client
self.partitioner_path = partitioner_path
self.buckets = buckets
self.log = log
self.got_buckets = got_buckets
self.time_boundary = time_boundary
ts = TimerService(interval, self.check_partition)
ts.setServiceParent(self)
ts.clock = clock
self._old_buckets = []
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
root = Root(config, app)
root = configRoot(root, config)
webservice = TCPServer(http_port, server.Site(root))
log.msg("Scrapyd web console available at http://localhost:%s/" % http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def __init__(self, kz_client, interval, partitioner_path, buckets,
time_boundary, log, got_buckets,
clock=None):
"""
:param log: a bound log
:param kz_client: txKazoo client
:param partitioner_path: ZooKeeper path, used for partitioning
:param buckets: iterable of buckets to distribute between
nodes. Ideally there should be at least as many elements as nodes
taking part in this partitioner. This should be a sequence of str.
:param time_boundary: time to wait for partitioning to stabilize.
:param got_buckets: Callable which will be called with a list of
buckets when buckets have been allocated to this node.
:param clock: clock to use for checking the buckets on an interval.
"""
MultiService.__init__(self)
self.kz_client = kz_client
self.partitioner_path = partitioner_path
self.buckets = buckets
self.log = log
self.got_buckets = got_buckets
self.time_boundary = time_boundary
ts = TimerService(interval, self.check_partition)
ts.setServiceParent(self)
ts.clock = clock
self._old_buckets = []
def __init__(self, reactor, config, log, clock=None, collect=None):
"""
Initialize the service by connecting to Cassandra and setting up
authenticator
:param reactor: Twisted reactor for connection purposes
:param dict config: All the config necessary to run the service.
Comes from config file
:param IReactorTime clock: Optional reactor for timer purpose
"""
self._client = connect_cass_servers(reactor, config['cassandra'])
self.log = log
self.reactor = reactor
self._divergent_groups = {}
self.divergent_timeout = get_in(
['metrics', 'divergent_timeout'], config, 3600)
self._service = TimerService(
get_in(['metrics', 'interval'], config, default=60),
collect or self.collect,
reactor,
config,
self.log,
client=self._client,
authenticator=generate_authenticator(reactor, config['identity']))
self._service.clock = clock or reactor
def startService(self):
self.log.info("Starting SQS poller {name} for URI {uri} (every {i}s)",
name=self.name,
uri=self.uri,
i=self.pollinterval)
self.timerService = TimerService(self.pollinterval, self.poll)
self.timerService.setServiceParent(self)
return super().startService()
def cb_start(result):
"""
Called when the client is ready for operation.
"""
self._timer_service = TimerService(
self.check_interval.total_seconds(), self._check_certs)
self._timer_service.clock = self._clock
self._timer_service.startService()
def startService(self):
"""
Start this service. This will start buckets partitioning
"""
self.kz_partition = self.kz_client.SetPartitioner(
self.zk_partition_path, set=set(self.buckets),
time_boundary=self.time_boundary)
TimerService.startService(self)
def __init__(self, reactor):
MultiService.__init__(self)
self._deployment_state = DeploymentState()
timer = TimerService(1, self._wipe_expired)
timer.clock = reactor
timer.setServiceParent(self)
self._information_wipers = pmap()
self._clock = reactor
def __init__(self, reactor):
MultiService.__init__(self)
self._deployment_state = DeploymentState()
timer = TimerService(1, self._wipe_expired)
timer.clock = reactor
timer.setServiceParent(self)
self._information_wipers = pmap()
self._clock = reactor
def __init__(self, server_process):
assert isinstance(server_process, ServerProcess), \
repr(server_process)
self.__server_process = server_process
self.__restore_timer = \
TimerService(WEB_RESTORE_PERIOD.total_seconds(),
exceptions_logged(logger)(callInThread),
self.__poll_restore_requests_in_thread)
def run(self):
self.m_connectorServer.begin(self.m_port)
self.m_timer = TimerService(1, self.timer)
self.m_timer.startService()
from twisted.internet import reactor
self.m_isRunning = True
reactor.run()
def __init__(self, basedir):
service.MultiService.__init__(self)
self.basedir = basedir
self.clients = {}
self.nicknames = {}
self.timer = TimerService(self.poll_interval, self.poll)
self.timer.setServiceParent(self)
def __init__(self, poller=None, step=300):
"""
"""
if poller is None:
raise RuntimeError('poller cannot be None')
if not issubclass(type(poller), IPoll):
raise TypeError('poller privided is not a subclass of IPoll')
self._poller = poller
TimerService.__init__(self, step=step, callable=self._poller.poll)
def run(self):
self.m_config.connect(self.configCallBack)
self.m_timer = TimerService(1, self.timer)
self.m_timer.startService()
#要放在最后一步
from twisted.internet import reactor
self.m_isRunning = True
logging.info("reactor run")
reactor.run()
def __init__(self, config, main_tub, control_tub, i2p_provider,
tor_provider, introducer_clients, storage_farm_broker):
"""
Use :func:`allmydata.client.create_client` to instantiate one of these.
"""
node.Node.__init__(self, config, main_tub, control_tub, i2p_provider,
tor_provider)
self._magic_folders = dict()
self.started_timestamp = time.time()
self.logSource = "Client"
self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy()
self.introducer_clients = introducer_clients
self.storage_broker = storage_farm_broker
self.init_stats_provider()
self.init_secrets()
self.init_node_key()
self.init_storage()
self.init_control()
self._key_generator = KeyGenerator()
key_gen_furl = config.get_config("client", "key_generator.furl", None)
if key_gen_furl:
log.msg("[client]key_generator.furl= is now ignored, see #2783")
self.init_client()
self.load_static_servers()
self.helper = None
if config.get_config("helper", "enabled", False, boolean=True):
if not self._is_tub_listening():
raise ValueError("config error: helper is enabled, but tub "
"is not listening ('tub.port=' is empty)")
self.init_helper()
self.init_ftp_server()
self.init_sftp_server()
self.init_magic_folder()
# If the node sees an exit_trigger file, it will poll every second to see
# whether the file still exists, and what its mtime is. If the file does not
# exist or has not been modified for a given timeout, the node will exit.
exit_trigger_file = config.get_config_path(self.EXIT_TRIGGER_FILE)
if os.path.exists(exit_trigger_file):
age = time.time() - os.stat(exit_trigger_file)[stat.ST_MTIME]
self.log("%s file noticed (%ds old), starting timer" %
(self.EXIT_TRIGGER_FILE, age))
exit_trigger = TimerService(1.0, self._check_exit_trigger,
exit_trigger_file)
exit_trigger.setServiceParent(self)
# this needs to happen last, so it can use getServiceNamed() to
# acquire references to StorageServer and other web-statusable things
webport = config.get_config("node", "web.port", None)
if webport:
self.init_web(webport) # strports string
def __init__(self, poller=None, step=60*5):
""" The magic here is calling the TimerService constructor (old
style class) to set the polling interval and specify the polling
function.
"""
if poller is None:
raise RuntimeError('poller cannot be None')
if not issubclass(type(poller), IPoll):
raise TypeError('poller provided is not a subclass of IPoll');
self._poller = poller
TimerService.__init__(self, step=step, callable=self._poller.poll)
def __init__(self, all_stations, base_config, base_dir, poll_length, file_pattern,
send_command, command_helper, complete_command, failed_command):
TimerService.__init__(self, poll_length, self.run_poll)
self.log = log
self.all_stations = all_stations
self.base_config = dict(base_config)
self.base_folder = base_dir
self.file_pattern = file_pattern
self.send_command = send_command
self.complete_command = complete_command
self.failed_command = failed_command
self.command_helper = command_helper
class StatsGatherer(Referenceable, service.MultiService):
implements(RIStatsGatherer)
poll_interval = 60
def __init__(self, basedir):
service.MultiService.__init__(self)
self.basedir = basedir
self.clients = {}
self.nicknames = {}
self.timer = TimerService(self.poll_interval, self.poll)
self.timer.setServiceParent(self)
def get_tubid(self, rref):
return rref.getRemoteTubID()
def remote_provide(self, provider, nickname):
tubid = self.get_tubid(provider)
if tubid == '<unauth>':
print "WARNING: failed to get tubid for %s (%s)" % (provider,
nickname)
# don't add to clients to poll (polluting data) don't care about disconnect
return
self.clients[tubid] = provider
self.nicknames[tubid] = nickname
def poll(self):
for tubid, client in self.clients.items():
nickname = self.nicknames.get(tubid)
d = client.callRemote('get_stats')
d.addCallbacks(
self.got_stats,
self.lost_client,
callbackArgs=(tubid, nickname),
errbackArgs=(tubid, ))
d.addErrback(self.log_client_error, tubid)
def lost_client(self, f, tubid):
# this is called lazily, when a get_stats request fails
del self.clients[tubid]
del self.nicknames[tubid]
f.trap(DeadReferenceError)
def log_client_error(self, f, tubid):
log.msg(
"StatsGatherer: error in get_stats(), peerid=%s" % tubid,
level=log.UNUSUAL,
failure=f)
def got_stats(self, stats, tubid, nickname):
raise NotImplementedError()
def __init__(self, reactor, interval, k8s, namespace, router):
TimerService.__init__(
self,
interval,
divert_errors_to_log(self._check_once, u"router-update"),
k8s,
namespace,
)
# This attribute controls the the reactor used by TimerService to set
# up the LoopingCall.
self.clock = reactor
self._router = router
def __init__(self, server_process):
assert isinstance(server_process, ServerProcess), \
repr(server_process)
self.server_process = server_process
self.lock = RLock()
self.__schedule_check_timer = \
TimerService(BACKUP_SCHEDULES_CHECK_PERIOD.total_seconds(),
self.__on_schedule_check_timer)
self.reread_cache()
def __init__(self, reactor, interval, k8s, namespace, router):
TimerService.__init__(
self,
interval,
divert_errors_to_log(self._check_once, u"router-update"),
k8s,
namespace,
)
# This attribute controls the the reactor used by TimerService to set
# up the LoopingCall.
self.clock = reactor
self._router = router
def __init__(self, fsnotify_manager):
"""Constructor.
@type fsnotify_manager: AbstractFSNotifyManager
"""
self.__inner_fsnotify_manager = fsnotify_manager
self.__watched_dir_to_cb = {} # mapping from path to callback
self.__implicitly_watched_dirs = set()
self.__poll_sync_dir_timer = \
TimerService(POLL_SYNC_DIR_FOR_SYMLINKS_PERIOD.total_seconds(),
lambda: callInThread(
self.__on_poll_sync_dir_timer))
self.__links_map_pair = LinksMapPair(links={}, reverse_links={})
def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider, introducer_clients,
storage_farm_broker):
"""
Use :func:`allmydata.client.create_client` to instantiate one of these.
"""
node.Node.__init__(self, config, main_tub, control_tub, i2p_provider, tor_provider)
self._magic_folders = dict()
self.started_timestamp = time.time()
self.logSource = "Client"
self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy()
self.introducer_clients = introducer_clients
self.storage_broker = storage_farm_broker
self.init_stats_provider()
self.init_secrets()
self.init_node_key()
self.init_storage()
self.init_control()
self._key_generator = KeyGenerator()
key_gen_furl = config.get_config("client", "key_generator.furl", None)
if key_gen_furl:
log.msg("[client]key_generator.furl= is now ignored, see #2783")
self.init_client()
self.load_static_servers()
self.helper = None
if config.get_config("helper", "enabled", False, boolean=True):
if not self._is_tub_listening():
raise ValueError("config error: helper is enabled, but tub "
"is not listening ('tub.port=' is empty)")
self.init_helper()
self.init_ftp_server()
self.init_sftp_server()
self.init_magic_folder()
# If the node sees an exit_trigger file, it will poll every second to see
# whether the file still exists, and what its mtime is. If the file does not
# exist or has not been modified for a given timeout, the node will exit.
exit_trigger_file = config.get_config_path(self.EXIT_TRIGGER_FILE)
if os.path.exists(exit_trigger_file):
age = time.time() - os.stat(exit_trigger_file)[stat.ST_MTIME]
self.log("%s file noticed (%ds old), starting timer" % (self.EXIT_TRIGGER_FILE, age))
exit_trigger = TimerService(1.0, self._check_exit_trigger, exit_trigger_file)
exit_trigger.setServiceParent(self)
# this needs to happen last, so it can use getServiceNamed() to
# acquire references to StorageServer and other web-statusable things
webport = config.get_config("node", "web.port", None)
if webport:
self.init_web(webport) # strports string
class StatsGatherer(Referenceable, service.MultiService):
implements(RIStatsGatherer)
poll_interval = 60
def __init__(self, basedir):
service.MultiService.__init__(self)
self.basedir = basedir
self.clients = {}
self.nicknames = {}
self.timer = TimerService(self.poll_interval, self.poll)
self.timer.setServiceParent(self)
def get_tubid(self, rref):
return rref.getRemoteTubID()
def remote_provide(self, provider, nickname):
tubid = self.get_tubid(provider)
if tubid == '<unauth>':
print "WARNING: failed to get tubid for %s (%s)" % (provider,
nickname)
# don't add to clients to poll (polluting data) don't care about disconnect
return
self.clients[tubid] = provider
self.nicknames[tubid] = nickname
def poll(self):
for tubid, client in self.clients.items():
nickname = self.nicknames.get(tubid)
d = client.callRemote('get_stats')
d.addCallbacks(self.got_stats,
self.lost_client,
callbackArgs=(tubid, nickname),
errbackArgs=(tubid, ))
d.addErrback(self.log_client_error, tubid)
def lost_client(self, f, tubid):
# this is called lazily, when a get_stats request fails
del self.clients[tubid]
del self.nicknames[tubid]
f.trap(DeadReferenceError)
def log_client_error(self, f, tubid):
log.msg("StatsGatherer: error in get_stats(), peerid=%s" % tubid,
level=log.UNUSUAL,
failure=f)
def got_stats(self, stats, tubid, nickname):
raise NotImplementedError()
def __init__(self,
agent,
url,
event_listeners,
interval=DEFAULT_INTERVAL,
state_store=None,
TimerService=TimerService,
coiterate=coiterate):
"""
:param agent: a :class:`twisted.web.client.Agent` to use to poll
:param url: the url to poll
:param event_listeners: listeners that handle a particular event
:type event_listeners: `iterable` of `callables` that take an event
as an argument
:param interval: how often to poll, given in seconds - defaults to 10
:type interval: ``int`` or ``float``
:param state_store: where to store the current polling state
:type state_store: :class:`otter.indexer.state.IStateStore` provider
:param TimerService: factory (not instance) that produces something
like a :class:`twisted.application.internet.TimerService` -
defaults to :class:`twisted.application.internet.TimerService`
(this parameter is mainly used for dependency injection for
testing)
:type TimerService: ``callable``
:param coiterate: function that is used to coiterate tasks - defaults
to :func:`twisted.internet.task.coiterate` - (this parameter is
mainly used for dependency injection for testing)
:type coiterate: ``callable``
"""
self._url = url
self._interval = interval
self._timer_service = TimerService(interval, self._do_poll)
self._next_url = None
self._agent = agent
self._state_store = state_store or DummyStateStore()
self._event_listeners = event_listeners
self._poll_timer = timer('FeedPollerService.poll.{0}'.format(url))
self._fetch_timer = timer('FeedPollerService.fetch.{0}'.format(url))
self._coiterate = coiterate
def __init__(self, basedir="."):
node.Node.__init__(self, basedir)
# All tub.registerReference must happen *after* we upcall, since
# that's what does tub.setLocation()
configutil.validate_config(self.config_fname, self.config,
_valid_config_sections())
self._magic_folder = None
self.started_timestamp = time.time()
self.logSource = "Client"
self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy()
self.init_introducer_clients()
self.init_stats_provider()
self.init_secrets()
self.init_node_key()
self.init_storage()
self.init_control()
self._key_generator = KeyGenerator()
key_gen_furl = self.get_config("client", "key_generator.furl", None)
if key_gen_furl:
log.msg("[client]key_generator.furl= is now ignored, see #2783")
self.init_client()
self.load_static_servers()
self.helper = None
if self.get_config("helper", "enabled", False, boolean=True):
if not self._tub_is_listening:
raise ValueError("config error: helper is enabled, but tub "
"is not listening ('tub.port=' is empty)")
self.init_helper()
self.init_ftp_server()
self.init_sftp_server()
self.init_magic_folder()
# If the node sees an exit_trigger file, it will poll every second to see
# whether the file still exists, and what its mtime is. If the file does not
# exist or has not been modified for a given timeout, the node will exit.
exit_trigger_file = os.path.join(self.basedir, self.EXIT_TRIGGER_FILE)
if os.path.exists(exit_trigger_file):
age = time.time() - os.stat(exit_trigger_file)[stat.ST_MTIME]
self.log("%s file noticed (%ds old), starting timer" %
(self.EXIT_TRIGGER_FILE, age))
exit_trigger = TimerService(1.0, self._check_exit_trigger,
exit_trigger_file)
exit_trigger.setServiceParent(self)
# this needs to happen last, so it can use getServiceNamed() to
# acquire references to StorageServer and other web-statusable things
webport = self.get_config("node", "web.port", None)
if webport:
self.init_web(webport) # strports string
def __init__(self, is_direction_incoming):
"""
@param is_direction_incoming: whether the measured direction
is incoming (otherwise outgoing).
@type is_direction_incoming: bool
"""
self.__direction_incoming = is_direction_incoming
self.__total_bytes = 0
self.__ticks = deque()
self.__ticks_lock = Lock()
self.__start_time = None
self.__timer_service = TimerService(
TRAFFIC_NOTIFICATION_PERIOD.total_seconds(),
self.__on_timer)
self.__started = False
def __init__(self, batchsize, interval, slv_client, clock=None):
"""
Initializes the scheduler service with batch size and interval
:param int batchsize: number of events to fetch on each iteration
:param int interval: time between each iteration
:param slv_client: a :class:`silverberg.client.CQLClient` or
:class:`silverberg.cluster.RoundRobinCassandraCluster` instance used to get lock
:param clock: An instance of IReactorTime provider that defaults to reactor if not provided
"""
from otter.models.cass import LOCK_TABLE_NAME
self.lock = BasicLock(slv_client, LOCK_TABLE_NAME, 'schedule', max_retry=0)
TimerService.__init__(self, interval, self.check_for_events, batchsize)
self.clock = clock
self.log = otter_log.bind(system='otter.scheduler')
def test_pickleTimerServiceNotPickleLoopFinished(self):
"""
When pickling L{internet.TimerService}, it won't pickle
L{internet.TimerService._loopFinished}.
"""
# We need a pickleable callable to test pickling TimerService. So we
# can't use self.timer
timer = TimerService(1, fakeTargetFunction)
timer.startService()
dumpedTimer = pickle.dumps(timer)
timer.stopService()
loadedTimer = pickle.loads(dumpedTimer)
nothing = object()
value = getattr(loadedTimer, "_loopFinished", nothing)
self.assertIdentical(nothing, value)
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
username = config.get('username', '')
password = config.get('password', '')
if username and password:
if ':' in username:
sys.exit("The `username` option contains illegal character ':', "
"check and update the configuration file of Scrapyd")
portal = Portal(PublicHTMLRealm(webcls(config, app)),
[StringCredentialsChecker(username, password)])
credential_factory = BasicCredentialFactory("Auth")
resource = HTTPAuthSessionWrapper(portal, [credential_factory])
log.msg("Basic authentication enabled")
else:
resource = webcls(config, app)
log.msg("Basic authentication disabled as either `username` or `password` is unset")
webservice = TCPServer(http_port, server.Site(resource), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def __init__(self, basedir="."):
node.Node.__init__(self, basedir)
# All tub.registerReference must happen *after* we upcall, since
# that's what does tub.setLocation()
configutil.validate_config(self.config_fname, self.config,
_valid_config_sections())
self._magic_folder = None
self.started_timestamp = time.time()
self.logSource="Client"
self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy()
self.init_introducer_clients()
self.init_stats_provider()
self.init_secrets()
self.init_node_key()
self.init_storage()
self.init_control()
self._key_generator = KeyGenerator()
key_gen_furl = self.get_config("client", "key_generator.furl", None)
if key_gen_furl:
log.msg("[client]key_generator.furl= is now ignored, see #2783")
self.init_client()
self.load_static_servers()
self.helper = None
if self.get_config("helper", "enabled", False, boolean=True):
if not self._tub_is_listening:
raise ValueError("config error: helper is enabled, but tub "
"is not listening ('tub.port=' is empty)")
self.init_helper()
self.init_ftp_server()
self.init_sftp_server()
self.init_magic_folder()
# If the node sees an exit_trigger file, it will poll every second to see
# whether the file still exists, and what its mtime is. If the file does not
# exist or has not been modified for a given timeout, the node will exit.
exit_trigger_file = os.path.join(self.basedir,
self.EXIT_TRIGGER_FILE)
if os.path.exists(exit_trigger_file):
age = time.time() - os.stat(exit_trigger_file)[stat.ST_MTIME]
self.log("%s file noticed (%ds old), starting timer" % (self.EXIT_TRIGGER_FILE, age))
exit_trigger = TimerService(1.0, self._check_exit_trigger, exit_trigger_file)
exit_trigger.setServiceParent(self)
# this needs to happen last, so it can use getServiceNamed() to
# acquire references to StorageServer and other web-statusable things
webport = self.get_config("node", "web.port", None)
if webport:
self.init_web(webport) # strports string
def makeService(config, channel_db="relay.sqlite", reactor=reactor):
increase_rlimits()
parent = MultiService()
channel_db = create_or_upgrade_channel_db(config["channel-db"])
usage_db = create_or_upgrade_usage_db(config["usage-db"])
log_file = (os.fdopen(int(config["log-fd"]), "w")
if config["log-fd"] is not None
else None)
server = make_server(channel_db,
allow_list=config["allow-list"],
advertise_version=config["advertise-version"],
signal_error=config["signal-error"],
blur_usage=config["blur-usage"],
usage_db=usage_db,
log_file=log_file,
)
server.setServiceParent(parent)
rebooted = time.time()
def expire():
now = time.time()
old = now - CHANNEL_EXPIRATION_TIME
server.prune_all_apps(now, old)
server.dump_stats(now, rebooted=rebooted)
TimerService(EXPIRATION_CHECK_PERIOD, expire).setServiceParent(parent)
log_requests = config["blur-usage"] is None
site = make_web_server(server, log_requests,
config["websocket-protocol-options"])
ep = endpoints.serverFromString(reactor, config["port"]) # to listen
StreamServerEndpointService(ep, site).setServiceParent(parent)
log.msg("websocket listening on ws://HOSTNAME:PORT/v1")
return parent
def __init__(self, jobs):
"""
jobs : dict
{job name : configuration options}
"""
super().__init__()
# For each job configuration, instantiate the requisite components
# and string everything together using (multi)service(s).
for name, cfg in jobs.items():
self.log.debug("configuring job {name}", name=name)
lw = Workspace(
sqlite.Engine(":memory:"),
fs.LocalDirectory(cfg["directory"], filters=cfg["filters"]),
)
# DEBUG
rw = Workspace(
sqlite.Engine(":memory:"),
fs.LocalDirectory("/tmp/wspace", filters=cfg["filters"]),
)
# END DEBUG
merger = TwoWayMerger(lw, rw)
trigger = TimerService(cfg.pop("frequency", .025), merger.sync)
self.addService(Job(name, merger, trigger))
def stopService(self):
if self.log_file is None:
return defer.succeed(None)
else:
self.log_file.close()
self.log_file = None
return TimerService.stopService(self)
def __init__(self, reactor, config, log, clock=None, collect=None):
"""
Initialize the service by connecting to Cassandra and setting up
authenticator
:param reactor: Twisted reactor for connection purposes
:param dict config: All the config necessary to run the service.
Comes from config file
:param IReactorTime clock: Optional reactor for timer purpose
"""
self._client = connect_cass_servers(reactor, config['cassandra'])
self.log = log
self.reactor = reactor
self._divergent_groups = {}
self.divergent_timeout = get_in(
['metrics', 'divergent_timeout'], config, 3600)
self._service = TimerService(
get_in(['metrics', 'interval'], config, default=60),
collect or self.collect,
reactor,
config,
self.log,
client=self._client,
authenticator=generate_authenticator(reactor, config['identity']))
self._service.clock = clock or reactor
def setUp(self):
"""
Set up a timer service to test.
"""
self.timer = TimerService(2, self.call)
self.clock = self.timer.clock = task.Clock()
self.deferred = Deferred()
def startService(self):
Service.startService(self)
self._registered = False
self._timer_service = TimerService(
self.check_interval.total_seconds(), self._check_certs)
self._timer_service.clock = self._clock
self._timer_service.startService()
def __init__(self, *args, **kwargs):
"""Constructor."""
super(UHostApp, self).__init__(*args, **kwargs)
# The manager that just tracks the FS
__internal_fsnotify_manager = FSNotifyManager()
# The manager that supports the symlinks on toplevel.
self.__fsnotify_manager = \
SyncDirFSNotifyProxy(fsnotify_manager=__internal_fsnotify_manager)
self.__syncer_starter = None
self.auto_start_sync = True
on_non_empty_cooling_down_to_store = \
lambda: logger_status_cooling_down_to_store.info(
'Some files cooled down to store recently',
extra={'status': True})
on_empty_cooling_down_to_store = \
lambda: logger_status_cooling_down_to_store.info(
'No more files cooling down to store',
extra={'status': False})
self.__cooling_down_to_store = \
EventfulDict(_on_non_empty_cb=on_non_empty_cooling_down_to_store,
_on_empty_cb=on_empty_cooling_down_to_store)
self.__cooling_down_to_store_lock = RLock()
self.__file_states_ready_to_write = {}
self.__file_states_ready_to_write_lock = Lock()
self.__ignore_file_paths = dict()
self.__ignore_file_paths_lock = RLock()
self.ux = ux.UX()
ux_handlers = [
ux.RestoreFSM(self.ux.handle_event),
ux.BackupFSM(self.ux.handle_event),
ux.SyncFSM(self.ux.handle_event),
ux.IdleFSM(self.ux.handle_event),
ux.NetworkConnectionFSM(self.ux.handle_event),
ux.OccupationHandler(self.ux.handle_event),
ux.UXEventForwarder()
]
for ux_handler in ux_handlers:
self.ux.add_handler(ux_handler)
self.__network_connection_is_working = True
logger.debug('Will attempt to create a dataset once every %s',
CREATE_DATASET_PERIOD)
logger.debug('Will publish connection state every %s',
PUBLISH_HOST_STATE_PERIOD)
self.__all_timer_services = [
# We should not create a dataset until the previous attempt
# of dataset creation has been completed.
# Thus DelayServiceForCallback rather than TimerService.
DelayServiceForCallback(period=CREATE_DATASET_PERIOD,
callback=self.__on_create_dataset_timer),
TimerService(PUBLISH_HOST_STATE_PERIOD.total_seconds(),
exceptions_logged(logger)(self.__publish_host_state))
]
def __init__(self, basedir):
service.MultiService.__init__(self)
self.basedir = basedir
self.clients = {}
self.nicknames = {}
self.timer = TimerService(self.poll_interval, self.poll)
self.timer.setServiceParent(self)
def __init__(self, server_process):
assert isinstance(server_process, ServerProcess), \
repr(server_process)
self.__server_process = server_process
self.__restore_timer = \
TimerService(WEB_RESTORE_PERIOD.total_seconds(),
exceptions_logged(logger)(callInThread),
self.__poll_restore_requests_in_thread)
def execute(self):
log.trace("Executing watcher.")
self._lastPrintedId = None
self._lastError = None
if self.config["interval"]:
svc = TimerService(self.config["interval"], self.showTimeline)
svc.setServiceParent(self)
# Since this runs ~forever, just return a Deferred that doesn't call
# back. A swift SIGINT will kill it.
d = Deferred()
else:
# Print it once and exit
d = self.showTimeline()
return d
def application(settings):
app = Application("Scraperd")
poller = QueuePoller(settings)
environment = Environment(settings)
app.setComponent(IPoller, poller)
app.setComponent(IEnvironment, environment)
laupath = settings.get('DAEMON_LAUNCHER', 'scraper.daemon.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(settings, app)
poll_interval = settings.getfloat('DAEMON_POLL_INTERVAL', 5)
timer = TimerService(poll_interval, poller.poll)
launcher.setServiceParent(app)
timer.setServiceParent(app)
return app
