def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(webcls(config, app)),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
host = get_host_ip(config)
redis_host = config.get('redis_host', 'localhost')
redis_port = config.get('redis_port', 6379)
redis_db = config.get('redis_db', 0)
redis_pool = redis.ConnectionPool(host=redis_host,
port=redis_port,
db=redis_db)
register_to_redis(config, redis_pool)
log.msg('Registering scrapyd [{}] to redis {}:{} at db {}'.format(
host, redis_host, redis_port, redis_db))
# log.msg('2018-11-03 10:10 am')
redis_interval = config.getfloat('redis_interval', 5)
register_timer = TimerService(redis_interval, register_to_redis, config,
redis_pool)
register_timer.setServiceParent(app)
return app
def start(self, callback, *args, **kwargs):
"""Start the timer with a given period."""
if args:
if kwargs:
self.timer = TimerService(self.period, callback, args, kwargs)
else:
self.timer = TimerService(self.period, callback, args)
else:
if kwargs:
self.timer = TimerService(self.period, callback, kwargs)
else:
self.timer = TimerService(self.period, callback)
self.timer.startService()
return
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg("Scrapyd web console available at http://%s:%s/" %
(bind_address, http_port))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def setup_selfheal_service(clock, config, dispatcher, health_checker, log):
"""
Setup selfheal timer service and return it.
:param clock: :obj:`IReactorTime` provider
:param dict config: Configuration dict containing selfheal info
:param dispatcher: Effect dispatcher
:param health_checker: ``HealthChecker`` object where SelfHeal's health
check will be added
:param log: :obj:`BoundLog` logger used by service
:return: selfheal service or None if relevant config is not found
:rtype: :obj:`IService`
"""
if "selfheal" not in config:
return None
interval = get_in(["selfheal", "interval"], config, no_default=True)
selfheal = SelfHeal(clock, dispatcher, config_value, interval, log)
func, lock = zk.locked_logged_func(dispatcher, "/selfheallock", log,
"selfheal-lock-acquired",
selfheal.setup)
health_checker.checks["selfheal"] = zk.create_health_check(lock)
sh_timer = TimerService(interval, func)
sh_timer.clock = clock
return sh_timer
def setUp(self):
"""
Set up a timer service to test.
"""
self.timer = TimerService(2, self.call)
self.clock = self.timer.clock = task.Clock()
self.deferred = Deferred()
def __init__(self, jobs):
"""
jobs : dict
{job name : configuration options}
"""
super().__init__()
# For each job configuration, instantiate the requisite components
# and string everything together using (multi)service(s).
for name, cfg in jobs.items():
self.log.debug("configuring job {name}", name=name)
lw = Workspace(
sqlite.Engine(":memory:"),
fs.LocalDirectory(cfg["directory"], filters=cfg["filters"]),
)
# DEBUG
rw = Workspace(
sqlite.Engine(":memory:"),
fs.LocalDirectory("/tmp/wspace", filters=cfg["filters"]),
)
# END DEBUG
merger = TwoWayMerger(lw, rw)
trigger = TimerService(cfg.pop("frequency", .025), merger.sync)
self.addService(Job(name, merger, trigger))
def __init__(self, reactor, config, log, clock=None, collect=None):
"""
Initialize the service by connecting to Cassandra and setting up
authenticator
:param reactor: Twisted reactor for connection purposes
:param dict config: All the config necessary to run the service.
Comes from config file
:param IReactorTime clock: Optional reactor for timer purpose
"""
self._client = connect_cass_servers(reactor, config['cassandra'])
self.log = log
self.reactor = reactor
self._divergent_groups = {}
self.divergent_timeout = get_in(
['metrics', 'divergent_timeout'], config, 3600)
self._service = TimerService(
get_in(['metrics', 'interval'], config, default=60),
collect or self.collect,
reactor,
config,
self.log,
client=self._client,
authenticator=generate_authenticator(reactor, config['identity']))
self._service.clock = clock or reactor
def __init__(self,
kz_client,
interval,
partitioner_path,
buckets,
time_boundary,
log,
got_buckets,
clock=None):
"""
:param log: a bound log
:param kz_client: txKazoo client
:param partitioner_path: ZooKeeper path, used for partitioning
:param buckets: iterable of buckets to distribute between
nodes. Ideally there should be at least as many elements as nodes
taking part in this partitioner. This should be a sequence of str.
:param time_boundary: time to wait for partitioning to stabilize.
:param got_buckets: Callable which will be called with a list of
buckets when buckets have been allocated to this node.
:param clock: clock to use for checking the buckets on an interval.
"""
MultiService.__init__(self)
self.kz_client = kz_client
self.partitioner_path = partitioner_path
self.buckets = buckets
self.log = log
self.got_buckets = got_buckets
self.time_boundary = time_boundary
ts = TimerService(interval, self.check_partition)
ts.setServiceParent(self)
ts.clock = clock
self._old_buckets = []
def makeService(config, channel_db="relay.sqlite", reactor=reactor):
increase_rlimits()
parent = MultiService()
channel_db = create_or_upgrade_channel_db(config["channel-db"])
usage_db = create_or_upgrade_usage_db(config["usage-db"])
log_file = (os.fdopen(int(config["log-fd"]), "w")
if config["log-fd"] is not None
else None)
server = make_server(channel_db,
allow_list=config["allow-list"],
advertise_version=config["advertise-version"],
signal_error=config["signal-error"],
blur_usage=config["blur-usage"],
usage_db=usage_db,
log_file=log_file,
)
server.setServiceParent(parent)
rebooted = time.time()
def expire():
now = time.time()
old = now - CHANNEL_EXPIRATION_TIME
server.prune_all_apps(now, old)
server.dump_stats(now, rebooted=rebooted)
TimerService(EXPIRATION_CHECK_PERIOD, expire).setServiceParent(parent)
log_requests = config["blur-usage"] is None
site = make_web_server(server, log_requests,
config["websocket-protocol-options"])
ep = endpoints.serverFromString(reactor, config["port"]) # to listen
StreamServerEndpointService(ep, site).setServiceParent(parent)
log.msg("websocket listening on ws://HOSTNAME:PORT/v1")
return parent
def application(config):
app = Application("Scrapyd")
http_port = int(environ.get('PORT', config.getint('http_port', 6800)))
config.cp.set('scrapyd', 'database_url', environ.get('DATABASE_URL'))
poller = Psycopg2QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = Psycopg2SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
log.msg("Scrapyd web console available at http://localhost:%s/ (HEROKU)" %
http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def __init__(
self,
password,
# On-Demand related stuff
instance_booter,
build_wait_timeout=60 * 10,
keepalive_interval=None,
# Generic stuff for the base class
max_builds=None,
notify_on_missing=[],
missing_timeout=60 * 20,
properties={},
locks=None,
):
# The buildslave name has already been supplied to the driver
# responsible for booting the node, so we use that attribute here.
name = instance_booter.driver.name
BuildSlave.__init__(self, name, password, max_builds,
notify_on_missing, missing_timeout, properties,
locks)
if build_wait_timeout < 0:
config.error("%s: %s: Can't wait for negative time." %
(self.__class__, name))
self.build_wait_timeout = build_wait_timeout
# Uggh
if keepalive_interval is not None:
self.keepalive_interval = keepalive_interval
self.building = set()
self.instance_booter = instance_booter
self.addService(TimerService(60, self.periodic))
def startService(self):
Service.startService(self)
self._registered = False
self._timer_service = TimerService(
self.check_interval.total_seconds(), self._check_certs)
self._timer_service.clock = self._clock
self._timer_service.startService()
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def __init__(self, clock=None, enable_monitoring=True):
# Order is very important here. First we set the clock to the passed-in
# reactor, so that unit tests can fake out the clock if necessary.
# Then we call super(). The superclass will set up the structures
# required to add parents to this service, which allows the remainder
# of this method to succeed. (And do so without the side-effect of
# executing calls that shouldn't be executed based on the desired
# reactor.)
self.clock = clock
super().__init__()
self.enable_monitoring = enable_monitoring
# The last successfully recorded interfaces.
self._recorded = None
self._monitored = frozenset()
self._monitoring_state = {}
self._monitoring_mdns = False
self._locked = False
# Use a named filesystem lock to prevent more than one monitoring
# service running on each host machine. This service attempts to
# acquire this lock on each loop, and then it holds the lock until the
# service stops.
self._lock = NetworksMonitoringLock()
# Set up child service to update interface.
self.interface_monitor = TimerService(self.interval,
self.updateInterfaces)
self.interface_monitor.setName("updateInterfaces")
self.interface_monitor.clock = self.clock
self.interface_monitor.setServiceParent(self)
self.beaconing_protocol = None
def __init__(self, *args, **kwargs):
"""Constructor."""
super(UHostApp, self).__init__(*args, **kwargs)
# The manager that just tracks the FS
__internal_fsnotify_manager = FSNotifyManager()
# The manager that supports the symlinks on toplevel.
self.__fsnotify_manager = \
SyncDirFSNotifyProxy(fsnotify_manager=__internal_fsnotify_manager)
self.__syncer_starter = None
self.auto_start_sync = True
on_non_empty_cooling_down_to_store = \
lambda: logger_status_cooling_down_to_store.info(
'Some files cooled down to store recently',
extra={'status': True})
on_empty_cooling_down_to_store = \
lambda: logger_status_cooling_down_to_store.info(
'No more files cooling down to store',
extra={'status': False})
self.__cooling_down_to_store = \
EventfulDict(_on_non_empty_cb=on_non_empty_cooling_down_to_store,
_on_empty_cb=on_empty_cooling_down_to_store)
self.__cooling_down_to_store_lock = RLock()
self.__file_states_ready_to_write = {}
self.__file_states_ready_to_write_lock = Lock()
self.__ignore_file_paths = dict()
self.__ignore_file_paths_lock = RLock()
self.ux = ux.UX()
ux_handlers = [
ux.RestoreFSM(self.ux.handle_event),
ux.BackupFSM(self.ux.handle_event),
ux.SyncFSM(self.ux.handle_event),
ux.IdleFSM(self.ux.handle_event),
ux.NetworkConnectionFSM(self.ux.handle_event),
ux.OccupationHandler(self.ux.handle_event),
ux.UXEventForwarder()
]
for ux_handler in ux_handlers:
self.ux.add_handler(ux_handler)
self.__network_connection_is_working = True
logger.debug('Will attempt to create a dataset once every %s',
CREATE_DATASET_PERIOD)
logger.debug('Will publish connection state every %s',
PUBLISH_HOST_STATE_PERIOD)
self.__all_timer_services = [
# We should not create a dataset until the previous attempt
# of dataset creation has been completed.
# Thus DelayServiceForCallback rather than TimerService.
DelayServiceForCallback(period=CREATE_DATASET_PERIOD,
callback=self.__on_create_dataset_timer),
TimerService(PUBLISH_HOST_STATE_PERIOD.total_seconds(),
exceptions_logged(logger)(self.__publish_host_state))
]
def __init__(self, reactor):
MultiService.__init__(self)
self._deployment_state = DeploymentState()
timer = TimerService(1, self._wipe_expired)
timer.clock = reactor
timer.setServiceParent(self)
self._information_wipers = pmap()
self._clock = reactor
def cb_start(result):
"""
Called when the client is ready for operation.
"""
self._timer_service = TimerService(
self.check_interval.total_seconds(), self._check_certs)
self._timer_service.clock = self._clock
self._timer_service.startService()
def startService(self):
self.log.info("Starting SQS poller {name} for URI {uri} (every {i}s)",
name=self.name,
uri=self.uri,
i=self.pollinterval)
self.timerService = TimerService(self.pollinterval, self.poll)
self.timerService.setServiceParent(self)
return super().startService()
def run(self):
self.m_connectorServer.begin(self.m_port)
self.m_timer = TimerService(1, self.timer)
self.m_timer.startService()
from twisted.internet import reactor
self.m_isRunning = True
reactor.run()
def __init__(self, server_process):
assert isinstance(server_process, ServerProcess), \
repr(server_process)
self.__server_process = server_process
self.__restore_timer = \
TimerService(WEB_RESTORE_PERIOD.total_seconds(),
exceptions_logged(logger)(callInThread),
self.__poll_restore_requests_in_thread)
def __init__(self, basedir):
service.MultiService.__init__(self)
self.basedir = basedir
self.clients = {}
self.nicknames = {}
self.timer = TimerService(self.poll_interval, self.poll)
self.timer.setServiceParent(self)
def run(self):
self.m_config.connect(self.configCallBack)
self.m_timer = TimerService(1, self.timer)
self.m_timer.startService()
#要放在最后一步
from twisted.internet import reactor
self.m_isRunning = True
logging.info("reactor run")
reactor.run()
def __init__(self, config, main_tub, control_tub, i2p_provider,
tor_provider, introducer_clients, storage_farm_broker):
"""
Use :func:`allmydata.client.create_client` to instantiate one of these.
"""
node.Node.__init__(self, config, main_tub, control_tub, i2p_provider,
tor_provider)
self._magic_folders = dict()
self.started_timestamp = time.time()
self.logSource = "Client"
self.encoding_params = self.DEFAULT_ENCODING_PARAMETERS.copy()
self.introducer_clients = introducer_clients
self.storage_broker = storage_farm_broker
self.init_stats_provider()
self.init_secrets()
self.init_node_key()
self.init_storage()
self.init_control()
self._key_generator = KeyGenerator()
key_gen_furl = config.get_config("client", "key_generator.furl", None)
if key_gen_furl:
log.msg("[client]key_generator.furl= is now ignored, see #2783")
self.init_client()
self.load_static_servers()
self.helper = None
if config.get_config("helper", "enabled", False, boolean=True):
if not self._is_tub_listening():
raise ValueError("config error: helper is enabled, but tub "
"is not listening ('tub.port=' is empty)")
self.init_helper()
self.init_ftp_server()
self.init_sftp_server()
self.init_magic_folder()
# If the node sees an exit_trigger file, it will poll every second to see
# whether the file still exists, and what its mtime is. If the file does not
# exist or has not been modified for a given timeout, the node will exit.
exit_trigger_file = config.get_config_path(self.EXIT_TRIGGER_FILE)
if os.path.exists(exit_trigger_file):
age = time.time() - os.stat(exit_trigger_file)[stat.ST_MTIME]
self.log("%s file noticed (%ds old), starting timer" %
(self.EXIT_TRIGGER_FILE, age))
exit_trigger = TimerService(1.0, self._check_exit_trigger,
exit_trigger_file)
exit_trigger.setServiceParent(self)
# this needs to happen last, so it can use getServiceNamed() to
# acquire references to StorageServer and other web-statusable things
webport = config.get_config("node", "web.port", None)
if webport:
self.init_web(webport) # strports string
def makeService(config, reactor=reactor):
increase_rlimits()
ep = endpoints.serverFromString(reactor, config["port"]) # to listen
log_file = (os.fdopen(int(config["log-fd"]), "w")
if config["log-fd"] is not None else None)
f = transit_server.Transit(blur_usage=config["blur-usage"],
log_file=log_file,
usage_db=config["usage-db"])
parent = MultiService()
StreamServerEndpointService(ep, f).setServiceParent(parent)
TimerService(5 * 60.0, f.timerUpdateStats).setServiceParent(parent)
return parent
def __init__(self):
service.MultiService.__init__(self)
# we don't use time.clock() here, because the constructor is run by
# the twistd parent process (as it loads the .tac file), whereas the
# rest of the program will be run by the child process, after twistd
# forks. Instead, set self.initial_cpu as soon as the reactor starts
# up.
self.initial_cpu = 0.0 # just in case
eventually(self._set_initial_cpu)
self.samples = []
# we provide 1min, 5min, and 15min moving averages
TimerService(self.POLL_INTERVAL, self.check).setServiceParent(self)
def __init__(self, server_process):
assert isinstance(server_process, ServerProcess), \
repr(server_process)
self.server_process = server_process
self.lock = RLock()
self.__schedule_check_timer = \
TimerService(BACKUP_SCHEDULES_CHECK_PERIOD.total_seconds(),
self.__on_schedule_check_timer)
self.reread_cache()
def application(config):
app = Application("Spider Platform Slave")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
performance = Performance()
egg_storage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IPerformance, performance)
app.setComponent(IEggStorage, egg_storage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
lau_path = config.get('launcher', 'slave.launcher.Launcher')
lau_cls = load_object(lau_path)
launcher = lau_cls(config, app)
web_path = config.get('web_root', 'slave.website.Root')
web_cls = load_object(web_path)
timer_queue = TimerService(poll_interval, poller.poll)
timer_performance = TimerService(1, performance.poll)
webservice = TCPServer(http_port, server.Site(web_cls(config, app)),
interface=bind_address)
log.msg(
format="Spider Platform Slave web console available at "
"http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer_queue.setServiceParent(app)
timer_performance.setServiceParent(app)
webservice.setServiceParent(app)
return app
def __init__(self, fsnotify_manager):
"""Constructor.
@type fsnotify_manager: AbstractFSNotifyManager
"""
self.__inner_fsnotify_manager = fsnotify_manager
self.__watched_dir_to_cb = {} # mapping from path to callback
self.__implicitly_watched_dirs = set()
self.__poll_sync_dir_timer = \
TimerService(POLL_SYNC_DIR_FOR_SYMLINKS_PERIOD.total_seconds(),
lambda: callInThread(
self.__on_poll_sync_dir_timer))
self.__links_map_pair = LinksMapPair(links={}, reverse_links={})
def build_service(reactor):
multi = MultiService()
StreamServerEndpointService(
TCP4ServerEndpoint(reactor, port),
server.Site(
wsgi.WSGIResource(reactor, reactor.getThreadPool(),
app), )).setServiceParent(multi)
logger = Logger()
TimerService(
# Run every 10 minutes
10 * 60,
lambda: deferToThread(check_for_revocation, cert_db, crtsh_checker)
.addErrback(lambda f: logger.failure(
"Error checking for revocation", f))).setServiceParent(multi)
TimerService(
60 * 60,
lambda: deferToThread(raw_app._update_lint_summaries).addErrback(
lambda f: logger.failure("Error updating cablint summaries", f
))).setServiceParent(multi)
return multi
def application(config):
app = Application("Scrapyd")
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config, app)
taskpoller = TaskPoller(config, app) #
app.setComponent(IPoller, poller)
app.setComponent(IPoller, taskpoller) #
timer = TimerService(poll_interval, poller.poll)
tasktimer = TimerService(poll_interval, taskpoller.poll) #
timer.setServiceParent(app)
tasktimer.setServiceParent(app) #
http_port = config.getint('http_port', 9090)
bind_address = config.get('bind_address', '127.0.0.1')
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
webservice.setServiceParent(app)
return app
