def application(config):
app = Application("Scrapyd")
http_port = int(environ.get('PORT', config.getint('http_port', 6800)))
config.cp.set('scrapyd', 'database_url', environ.get('DATABASE_URL'))
poller = Psycopg2QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = Psycopg2SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
log.msg("Scrapyd web console available at http://localhost:%s/ (HEROKU)"
% http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def build(cls, root_service):
if not settings.ENABLE_MANHOLE:
return
factory = createManholeListener()
service = TCPServer(settings.MANHOLE_PORT, factory, interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
def application(config):
app = Application("ScrapydArt")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port, server.Site(webcls(config, app)), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def get_application(config):
app = Application('Scrapyd')
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd_mongodb.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg('http://%(bind_address)s:%(http_port)s/' % {
'bind_address': bind_address,
'http_port': http_port
})
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def createCacheService(config):
from carbon.cache import MetricCache
from carbon.conf import settings
from carbon.protocols import CacheManagementHandler
# Configure application components
events.metricReceived.addHandler(MetricCache.store)
root_service = createBaseService(config)
factory = ServerFactory()
factory.protocol = CacheManagementHandler
service = TCPServer(int(settings.CACHE_QUERY_PORT),
factory,
interface=settings.CACHE_QUERY_INTERFACE)
service.setServiceParent(root_service)
# have to import this *after* settings are defined
from carbon.writer import WriterService
service = WriterService()
service.setServiceParent(root_service)
if settings.USE_FLOW_CONTROL:
events.cacheFull.addHandler(events.pauseReceivingMetrics)
events.cacheSpaceAvailable.addHandler(events.resumeReceivingMetrics)
return root_service
def application(config):
app = Application("Scrapyd")
http_port = int(environ.get('PORT', config.getint('http_port', 6800)))
config.cp.set('scrapyd', 'database_url', environ.get('DATABASE_URL'))
poller = Psycopg2QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = Psycopg2SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
log.msg("Scrapyd web console available at http://localhost:%s/ (HEROKU)" %
http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def get_application(config):
app = Application('Scrapyd')
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd_mongodb.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(
http_port, server.Site(Root(config, app)),
interface=bind_address)
log.msg('http://%(bind_address)s:%(http_port)s/' % {'bind_address':bind_address, 'http_port':http_port})
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
root = Root(config, app)
root = configRoot(root, config)
webservice = TCPServer(http_port, server.Site(root))
log.msg("Scrapyd web console available at http://localhost:%s/" % http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg("Scrapyd web console available at http://%s:%s/" % (bind_address, http_port))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def get_application(arguments):
ServiceRoot = load_object(settings.SERVICE_ROOT)
site = Site(ServiceRoot())
application = Application('scrapyrt')
server = TCPServer(arguments.port, site, interface=arguments.ip)
server.setServiceParent(application)
return application
def get_application(arguments):
ServiceRoot = load_object(settings.SERVICE_ROOT)
site = Site(ServiceRoot())
application = Application('scrapyrt')
server = TCPServer(arguments.port, site, interface=arguments.ip)
server.setServiceParent(application)
return application
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
portal = Portal(PublicHTMLRealm(config, app), [FilePasswordDB(str(config.get('passwd', '')))])
credentialFactory = DigestCredentialFactory("md5", "Go away")
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(HTTPAuthSessionWrapper(portal, [credentialFactory])))
log.msg("Scrapyd web console available at http://localhost:%s/" % http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def createCacheService(config):
from carbon.cache import MetricCache
from carbon.conf import settings
from carbon.protocols import CacheManagementHandler
# Configure application components
events.metricReceived.addHandler(MetricCache.store)
root_service = createBaseService(config)
factory = ServerFactory()
factory.protocol = CacheManagementHandler
service = TCPServer(int(settings.CACHE_QUERY_PORT), factory,
interface=settings.CACHE_QUERY_INTERFACE)
service.setServiceParent(root_service)
# have to import this *after* settings are defined
from carbon.writer import WriterService
service = WriterService()
service.setServiceParent(root_service)
if settings.USE_FLOW_CONTROL:
events.cacheFull.addHandler(events.pauseReceivingMetrics)
events.cacheSpaceAvailable.addHandler(events.resumeReceivingMetrics)
return root_service
def makeService(self, options):
"""
Create a service which will run a scram server.
@param options: mapping of configuration
"""
from scram.network import ScramFactory
from scram.world import (TCP_SERVICE_NAME, SCRAM_SERVICE_NAME, ScramService, World)
from twisted.internet import reactor
from twisted.application.service import MultiService
from twisted.protocols.policies import TrafficLoggingFactory
world = World(granularity=60, platformClock=reactor)
service = MultiService()
factory = ScramFactory(world)
if options['log-directory'] is not None:
factory = TrafficLoggingFactory(
factory, join(options['log-directory'], 'scram'))
tcp = TCPServer(options['port'], factory, interface='127.0.0.1')
tcp.setName(TCP_SERVICE_NAME)
tcp.setServiceParent(service)
scram = ScramService(world)
scram.setName(SCRAM_SERVICE_NAME)
scram.setServiceParent(service)
return service
def application(config, components=interfaces):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
for interface, key in interfaces:
path = config.get(key)
cls = load_object(path)
component = cls(config)
app.setComponent(interface, component)
poller = component
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
poll_every = config.getint("poll_every", 5)
timer = TimerService(poll_every, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg("Scrapyd web console available at http://%s:%s/" %
(bind_address, http_port))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def makeService(self, options):
"""
Create a service which will run a scram server.
@param options: mapping of configuration
"""
from scram.network import ScramFactory
from scram.world import (TCP_SERVICE_NAME, SCRAM_SERVICE_NAME,
ScramService, World)
from twisted.internet import reactor
from twisted.application.service import MultiService
from twisted.protocols.policies import TrafficLoggingFactory
world = World(granularity=60, platformClock=reactor)
service = MultiService()
factory = ScramFactory(world)
if options['log-directory'] is not None:
factory = TrafficLoggingFactory(
factory, join(options['log-directory'], 'scram'))
tcp = TCPServer(options['port'], factory, interface='127.0.0.1')
tcp.setName(TCP_SERVICE_NAME)
tcp.setServiceParent(service)
scram = ScramService(world)
scram.setName(SCRAM_SERVICE_NAME)
scram.setServiceParent(service)
return service
def registry(self):
root = NestRoot(self.app, debug=False)
for (path, service) in self.webservices.items():
root.putChild(path.encode('utf-8'), service(root))
server = TCPServer(self.port, Site(root), interface=self.host)
server.setServiceParent(self.app)
return self.app
def build(cls, root_service):
if not settings.ENABLE_MANHOLE:
return
factory = createManholeListener()
service = TCPServer(settings.MANHOLE_PORT,
factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
def makeService(self, options):
config = options
s = MultiService()
from cap_twisted import service as mainrpc
serverfactory = server.Site(mainrpc.MainRpc(config["master"],config["work_dir"]))
slave_service = TCPServer(9913, serverfactory, interface=config["host"])
slave_service.setServiceParent(s)
return s
def makeService(config):
event_db = Event_DB(config['eventdb'])
LoopingCall(event_db.prune, MAX_AGE).start(PRUNE_INTERVAL)
broker_service = MultiService()
if config['broadcast']:
broadcaster_factory = VOEventBroadcasterFactory(
config["local-ivo"], config['broadcast-test-interval']
)
if log.LEVEL >= log.Levels.INFO: broadcaster_factory.noisy = False
broadcaster_service = TCPServer(
config['broadcast-port'],
broadcaster_factory
)
broadcaster_service.setName("Broadcaster")
broadcaster_service.setServiceParent(broker_service)
# If we're running a broadcast, we will rebroadcast any events we
# receive to it.
config['handlers'].append(EventRelay(broadcaster_factory))
if config['receive']:
receiver_factory = VOEventReceiverFactory(
local_ivo=config['local-ivo'],
validators=[
CheckPreviouslySeen(event_db),
CheckSchema(
os.path.join(comet.__path__[0], "schema/VOEvent-v2.0.xsd")
),
CheckIVORN()
],
handlers=config['handlers']
)
if log.LEVEL >= log.Levels.INFO: receiver_factory.noisy = False
whitelisting_factory = WhitelistingFactory(receiver_factory, config['whitelist'])
if log.LEVEL >= log.Levels.INFO: whitelisting_factory.noisy = False
receiver_service = TCPServer(config['receive-port'], whitelisting_factory)
receiver_service.setName("Receiver")
receiver_service.setServiceParent(broker_service)
for host, port in config["remotes"]:
subscriber_factory = VOEventSubscriberFactory(
local_ivo=config["local-ivo"],
validators=[CheckPreviouslySeen(event_db)],
handlers=config['handlers'],
filters=config['filters']
)
if log.LEVEL >= log.Levels.INFO: subscriber_factory.noisy = False
remote_service = TCPClient(host, port, subscriber_factory)
remote_service.setName("Remote %s:%d" % (host, port))
remote_service.setServiceParent(broker_service)
if not broker_service.services:
reactor.callWhenRunning(log.warning, "No services requested; stopping.")
reactor.callWhenRunning(reactor.stop)
return broker_service
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(webcls(config, app)),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
host = get_host_ip(config)
redis_host = config.get('redis_host', 'localhost')
redis_port = config.get('redis_port', 6379)
redis_db = config.get('redis_db', 0)
redis_pool = redis.ConnectionPool(host=redis_host,
port=redis_port,
db=redis_db)
register_to_redis(config, redis_pool)
log.msg('Registering scrapyd [{}] to redis {}:{} at db {}'.format(
host, redis_host, redis_port, redis_db))
# log.msg('2018-11-03 10:10 am')
redis_interval = config.getfloat('redis_interval', 5)
register_timer = TimerService(redis_interval, register_to_redis, config,
redis_pool)
register_timer.setServiceParent(app)
return app
def init(application):
global PORT, MATRIX_REMOTE
root = ApplicationHome()
root.putChild("rooms", RoomsPage())
root.putChild("users", UsersPage())
root.putChild("transactions", TransactionsPage())
factory = Site(root)
j = TCPServer(PORT, factory)
j.setServiceParent(application)
if MATRIX_REMOTE is not None:
startTunnelToMatrix()
def makeService(self, options):
top_service = service.MultiService()
server_service = ServerService(options["map"], int(options["numplayers"]))
server_service.setServiceParent(top_service)
server_factory = MyServerFactory(server_service)
server = TCPServer(int(options["port"]), server_factory)
server.setServiceParent(top_service)
return top_service
def create_status_service(storage, parent_service, port, user_id=0, ssl_context_factory=None):
"""Create the status service."""
root = resource.Resource()
root.putChild("status", _Status(storage, user_id))
root.putChild("+meliae", MeliaeResource())
root.putChild("+gc-stats", GCResource())
site = server.Site(root)
if ssl_context_factory is None:
service = TCPServer(port, site)
else:
service = SSLServer(port, site, ssl_context_factory)
service.setServiceParent(parent_service)
return service
def addProtocol(self, protocol, port):
self.log.info(
"Setting service for protocol {protocol!r} on port {port}...",
protocol=protocol, port=port,
)
# TCP Service
tcpFactory = SpawningInheritingProtocolFactory(
self.dispatcher, self.spawningService, protocol
)
tcpService = TCPServer(port, tcpFactory)
tcpService.setServiceParent(self)
def create_status_service(storage, parent_service, port,
user_id=0, ssl_context_factory=None):
"""Create the status service."""
root = resource.Resource()
root.putChild('status', _Status(storage, user_id))
root.putChild('+meliae', MeliaeResource())
root.putChild('+gc-stats', GCResource())
site = server.Site(root)
if ssl_context_factory is None:
service = TCPServer(port, site)
else:
service = SSLServer(port, site, ssl_context_factory)
service.setServiceParent(parent_service)
return service
def createCacheService(config):
from carbon.cache import MetricCache
from carbon.conf import settings
from carbon.protocols import CacheManagementHandler
# Configure application components
events.metricReceived.addHandler(MetricCache.store)
root_service = createBaseService(config)
factory = ServerFactory()
factory.protocol = CacheManagementHandler
service = TCPServer(int(settings.CACHE_QUERY_PORT), factory,
interface=settings.CACHE_QUERY_INTERFACE)
service.setServiceParent(root_service)
# have to import this *after* settings are defined
from carbon.writer import WriterService
service = WriterService()
service.setServiceParent(root_service)
# use_amqp_pub = settings.get("ENABLE_AMQP_PUB", False)
# if use_amqp_pub:
# from carbon import amqp_pub
# amqp_pub_host = settings.get("AMQP_PUB_HOST", "localhost")
# amqp_pub_port = settings.get("AMQP_PUB_PORT", 5672)
# amqp_pub_user = settings.get("AMQP_PUB_USER", "guest")
# amqp_pub_password = settings.get("AMQP_PUB_PASSWORD", "guest")
# amqp_pub_verbose = settings.get("AMQP_PUB_VERBOSE", False)
# amqp_pub_vhost = settings.get("AMQP_PUB_VHOST", "/")
# amqp_pub_spec = settings.get("AMQP_PUB_SPEC", None)
# amqp_pub_exchange_name = settings.get("AMQP_EXCHANGE", "graphite")
# factory = amqp_pub.createAMQPPublisher(
# amqp_pub_user, amqp_pub_password,
# vhost=amqp_pub_vhost, spec=amqp_pub_spec,
# exchange_name=amqp_pub_exchange_name,
# verbose=amqp_pub_verbose)
# service = TCPClient(amqp_pub_host, int(amqp_pub_port), factory)
# service.setServiceParent(root_service)
# from carbon.amqp_pub import ProducerService
# service = ProducerService()
# service.setServiceParent(root_service)
if settings.USE_FLOW_CONTROL:
events.cacheFull.addHandler(events.pauseReceivingMetrics)
events.cacheSpaceAvailable.addHandler(events.resumeReceivingMetrics)
return root_service
def setupReceivers(root_service, settings):
from carbon.protocols import MetricLineReceiver, MetricPickleReceiver, MetricDatagramReceiver
for protocol, interface, port in [
(MetricLineReceiver, settings.LINE_RECEIVER_INTERFACE, settings.LINE_RECEIVER_PORT),
(MetricPickleReceiver, settings.PICKLE_RECEIVER_INTERFACE, settings.PICKLE_RECEIVER_PORT)
]:
if port:
factory = ServerFactory()
factory.protocol = protocol
service = TCPServer(port, factory, interface=interface)
service.setServiceParent(root_service)
if settings.ENABLE_UDP_LISTENER:
service = UDPServer(int(settings.UDP_RECEIVER_PORT),
MetricDatagramReceiver(),
interface=settings.UDP_RECEIVER_INTERFACE)
service.setServiceParent(root_service)
if settings.ENABLE_AMQP:
from carbon import amqp_listener
amqp_host = settings.AMQP_HOST
amqp_port = settings.AMQP_PORT
amqp_user = settings.AMQP_USER
amqp_password = settings.AMQP_PASSWORD
amqp_verbose = settings.AMQP_VERBOSE
amqp_vhost = settings.AMQP_VHOST
amqp_spec = settings.AMQP_SPEC
amqp_exchange_name = settings.AMQP_EXCHANGE
factory = amqp_listener.createAMQPListener(
amqp_user,
amqp_password,
vhost=amqp_vhost,
spec=amqp_spec,
exchange_name=amqp_exchange_name,
verbose=amqp_verbose)
service = TCPClient(amqp_host, amqp_port, factory)
service.setServiceParent(root_service)
if settings.ENABLE_MANHOLE:
from carbon import manhole
# Configure application components
if settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
elif settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
RuleManager.read_from(settings["aggregation-rules"])
router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR)
elif settings.RELAY_METHOD == 'remove-node-consistent-hashing':
router = RemoveNodeConsistentHashingRouter(settings.REPLICATION_FACTOR, settings.REMOVE_NODE_INDEX)
factory = manhole.createManholeListener()
service = TCPServer(
settings.MANHOLE_PORT,
factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
def makeService(self, options):
"""Construct a service."""
services = MultiService()
config_file = options["config-file"]
config = Config.load(config_file)
log_service = LogService(config["logfile"])
log_service.setServiceParent(services)
oops_config = config["oops"]
oops_dir = oops_config["directory"]
oops_reporter = oops_config["reporter"]
oops_service = OOPSService(log_service, oops_dir, oops_reporter)
oops_service.setServiceParent(services)
frontend_config = config["frontend"]
frontend_port = frontend_config["port"]
frontend_prefix = frontend_config["prefix"]
frontend_interface = frontend_config["interface"]
frontend_manager = DeprecatedQueueManager(frontend_prefix)
broker_config = config["broker"]
broker_port = broker_config["port"]
broker_host = broker_config["host"]
broker_username = broker_config["username"]
broker_password = broker_config["password"]
broker_vhost = broker_config["vhost"]
cb_connected = frontend_manager.connected
cb_disconnected = frontend_manager.disconnected
cb_failed = lambda connector_and_reason: (log.err(
connector_and_reason[1], "Connection failed"))
client_factory = AMQFactory(broker_username, broker_password,
broker_vhost, cb_connected,
cb_disconnected, cb_failed)
client_service = TCPClient(broker_host, broker_port, client_factory)
client_service.setName("amqp")
client_service.setServiceParent(services)
frontend_resource = FrontEndAjax(frontend_manager)
frontend_service = TCPServer(frontend_port,
Site(frontend_resource),
interface=frontend_interface)
frontend_service.setName("frontend")
frontend_service.setServiceParent(services)
return services
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
username = config.get('username', '')
password = config.get('password', '')
if username and password:
if ':' in username:
sys.exit("The `username` option contains illegal character ':', "
"check and update the configuration file of Scrapyd")
portal = Portal(PublicHTMLRealm(webcls(config, app)),
[StringCredentialsChecker(username, password)])
credential_factory = BasicCredentialFactory("Auth")
resource = HTTPAuthSessionWrapper(portal, [credential_factory])
log.msg("Basic authentication enabled")
else:
resource = webcls(config, app)
log.msg("Basic authentication disabled as either `username` or `password` is unset")
webservice = TCPServer(http_port, server.Site(resource), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Flowder")
app_id = config.get('app_id', 'fw0')
logfile = config.get('logfile', '/var/log/flowder.log')
loglevel = config.get('loglevel', 'info')
db_file = config.get('db_file', 'flowder')
rest_port = config.getint('rest_port', 4000)
rest_bind = config.get('rest_bind', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 1)
poll_size = config.getint("poll_size", 5)
signalmanager = SignalManager()
app.setComponent(ISignalManager, signalmanager)
fetcher = FetcherService(config)
fetcher.setServiceParent(app)
poller = QueuePoller(app, poll_size)
poller.setServiceParent(app)
db_file = '%s.db' % db_file
task_storage = FileDownloaderTaskStorage(app, db_file)
task_storage.setServiceParent(app)
timer = TimerService(poll_interval, poller.poll)
timer.setServiceParent(app)
scheduler = TaskScheduler(config, app)
scheduler.setServiceParent(app)
laupath = config.get('launcher', 'flowder.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(app, config)
launcher.setServiceParent(app)
restService = TCPServer(rest_port,
server.Site(Root(app, config)),
interface=rest_bind)
restService.setServiceParent(app)
amqp_publisher = AmqpService(app, config)
amqp_publisher.setServiceParent(app)
log.msg("Starting Flowder services (;-)")
return app
def setupWriterProcessor(root_service, settings):
from carbon import cache # Register CacheFeedingProcessor
from carbon.protocols import CacheManagementHandler
from carbon.writer import WriterService
from carbon import events
factory = ServerFactory()
factory.protocol = CacheManagementHandler
service = TCPServer(settings.CACHE_QUERY_PORT, factory, interface=settings.CACHE_QUERY_INTERFACE)
service.setServiceParent(root_service)
writer_service = WriterService()
writer_service.setServiceParent(root_service)
if settings.USE_FLOW_CONTROL:
events.cacheFull.addHandler(events.pauseReceivingMetrics)
events.cacheSpaceAvailable.addHandler(events.resumeReceivingMetrics)
def makeService(self, options):
config = valid_config()
s = MultiService()
from crondeamon.slave import service as subrpc
serverfactory = server.Site(subrpc.MainRpc())
slave_service = TCPServer(int(config["slaveport"]),
serverfactory,
interface=config["host"])
slave_service.setServiceParent(s)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cap.settings")
from django.core.handlers.wsgi import WSGIHandler
application = WSGIHandler()
resource = WSGIResource(reactor, reactor.getThreadPool(), application)
ui_service = TCPServer(int(config["uiport"]),
server.Site(resource),
interface=config["host"])
ui_service.setServiceParent(s)
return s
def setupWriterProcessor(root_service, settings):
from carbon import cache # NOQA Register CacheFeedingProcessor
from carbon.protocols import CacheManagementHandler
from carbon.writer import WriterService
factory = ServerFactory()
factory.protocol = CacheManagementHandler
service = TCPServer(settings.CACHE_QUERY_PORT,
factory,
interface=settings.CACHE_QUERY_INTERFACE)
service.setServiceParent(root_service)
writer_service = WriterService()
writer_service.setServiceParent(root_service)
if settings.USE_FLOW_CONTROL:
events.cacheFull.addHandler(events.pauseReceivingMetrics)
events.cacheSpaceAvailable.addHandler(events.resumeReceivingMetrics)
def application(config):
"""
提供http服务并启动应用
:param config:
:return:
"""
app = Application("engine")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
# metrics = MetricsReporter(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
# app.setComponent(IMetrics, metrics)
laupath = config.get('launcher', 'engine.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get('webroot', 'engine.website.Root')
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(webcls(config, app)),
interface=bind_address)
log.msg(
format="DtCrawlEngine 访问地址 at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
schedpath = config.get('scheduler', 'scrapyd.scheduler.SpiderScheduler')
schedCls = load_object(schedpath)
scheduler = schedCls(config, app)
environment = Environment(config)
pubsub_path = config.get('pubsub', 'scrapyd.pubsub.BasePubSub')
pubsubCls = load_object(pubsub_path)
pubsub = pubsubCls(config, app)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
app.setComponent(IPubSub, pubsub)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
pubsub.setServiceParent(app)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def makeService(config):
'''
Create service tree. Only one tracker can be used for every protocol!
@param config: instance of an Options class with configuration parameters.
@type config: C{twisted.python.usage.Options}
@return: service collection
@rtype: C{twisted.application.service.IServiceCollection}
'''
from gorynych.receiver.factories import ReceivingFactory
from gorynych.receiver import protocols, parsers
from gorynych.receiver.receiver import ReceiverRabbitQueue, ReceiverService, AuditFileLog
# Set up application.
application = service.Application("ReceiverServer")
sc = service.IServiceCollection(application)
# Prepare receiver.
audit_log = AuditFileLog('audit_log')
sender = ReceiverRabbitQueue(host='localhost', port=5672,
exchange='receiver', exchange_type='fanout')
parser = getattr(parsers, config['tracker'])()
sender.setName('RabbitMQReceiverService')
sender.setServiceParent(sc)
receiver_service = ReceiverService(sender, audit_log, parser)
receiver_service.setName('ReceiverService')
receiver_service.setServiceParent(sc)
if 'tcp' in config['protocols']:
receiver_server = TCPServer(config['port'],
ReceivingFactory(receiver_service))
protocol = getattr(protocols,
'_'.join((config['tracker'], 'tcp', 'protocol')))
receiver_server.args[1].protocol = protocol
receiver_server.setName(config['tracker'] + '_tcp')
receiver_server.setServiceParent(sc)
if 'udp' in config['protocols']:
protocol = getattr(protocols,
'_'.join((config['tracker'], 'udp', 'protocol')))(receiver_service)
receiver_server = UDPServer(config['port'], protocol)
receiver_server.setName(config['tracker'] + '_udp')
receiver_server.setServiceParent(sc)
return sc
def makeService(config, reactor=reactor):
parent = MultiService()
basedir = FilePath(os.path.expanduser(config["basedir"]))
basedir.makedirs(ignoreExistingDirectory=True)
basedir.chmod(0o700)
data = Data(basedir.child("config.json"))
dns_server = DNSServerFactory(verbose=0)
s1 = UDPServer(int(config["dns-port"]), dns.DNSDatagramProtocol(dns_server),
interface=config["dns-interface"])
s1.setServiceParent(parent)
s2 = TCPServer(int(config["dns-port"]), dns_server,
interface=config["dns-interface"])
s2.setServiceParent(parent)
s = Server(data, dns_server)
s.update_records()
certFile = basedir.child("tub.data").path
#furlFile = basedir.child("server.furl").path
t = Tub(certFile=certFile)
t.setOption("keepaliveTimeout", 60) # ping after 60s of idle
t.setOption("disconnectTimeout", 5*60) # disconnect/reconnect after 5m
#t.setOption("logLocalFailures", True)
#t.setOption("logRemoteFailures", True)
#t.unsafeTracebacks = True
fp = config["foolscap-port"]
if not fp.startswith("tcp:"):
raise usage.UsageError("I don't know how to handle non-tcp foolscap-port=")
port = int(fp.split(":")[1])
assert port > 1
t.listenOn(fp)
t.setLocation("tcp:%s:%d" % (config["hostname"], port))
c = Controller(data, s)
cf = t.registerReference(c, furlFile=basedir.child("controller.furl").path)
furl_prefix = cf[:cf.rfind("/")+1]
c.set_furl_prefix(furl_prefix)
t.registerNameLookupHandler(c.lookup)
t.setServiceParent(parent)
return parent
def makeService(self, options):
"""
Create a service which will run a Gam3 server.
@param options: mapping of configuration
"""
from pygame.image import load
from gam3.network import Gam3Factory
from gam3.world import (
TCP_SERVICE_NAME, GAM3_SERVICE_NAME, Gam3Service, World)
from game.terrain import loadTerrainFromString, loadTerrainFromSurface
from twisted.python.filepath import FilePath
from twisted.internet import reactor
from twisted.application.service import MultiService
from twisted.protocols.policies import TrafficLoggingFactory
world = World(granularity=100, platformClock=reactor)
terrain = options['terrain']
if terrain:
if terrain.endswith('.png'):
voxels = loadTerrainFromSurface(load(terrain)).voxels
else:
raw = FilePath(terrain).getContent()
voxels = loadTerrainFromString(raw)
world.terrain.set(0, 0, 0, voxels)
service = MultiService()
factory = Gam3Factory(world)
if options['log-directory'] is not None:
factory = TrafficLoggingFactory(
factory, join(options['log-directory'], 'gam3'))
tcp = TCPServer(options['port'], factory)
tcp.setName(TCP_SERVICE_NAME)
tcp.setServiceParent(service)
gam3 = Gam3Service(world)
gam3.setName(GAM3_SERVICE_NAME)
gam3.setServiceParent(service)
return service
def makeService(self, options):
top_service = MultiService()
config_file = os.path.expanduser(options['config'])
config = Config([config_file])
#-----------------------------------------------------------------------
# Set up the controller
#-----------------------------------------------------------------------
try:
controller = Controller(config)
controller.setServiceParent(top_service)
except Exception as e:
log.msg(format="Unable to set up the controller: %(reason)s",
reason=exc_repr(e),
logLevel=logging.ERROR)
return top_service
#-----------------------------------------------------------------------
# Set up the web server
#-----------------------------------------------------------------------
try:
web_app = WebApp(config, controller)
site = server.Site(web_app)
interface = config.get_string('web', 'interface')
port = config.get_int('web', 'port')
web_server = TCPServer(port, site, interface=interface)
web_server.setServiceParent(top_service)
if ':' in interface:
interface = '[{}]'.format(interface)
log.msg(format="PiPilot web interface is available at "
"http://%(interface)s:%(port)s/",
interface=interface,
port=port)
except Exception as e:
log.msg(format="Scrapy-Do web interface could not have been "
"configured: %(reason)s",
reason=exc_repr(e),
logLevel=logging.ERROR)
return top_service
return top_service
def make_application(remoteHost, remotePort, listenPort, bindAddress):
from twisted.application.internet import TCPServer
from twisted.application.service import Application
from twisted.internet import reactor
applicationName = 'xoauth2relay'
application = Application(applicationName)
from .oauth2 import CredentialsStore
from .oauth2 import OAuth2AuthenticatorRefreshingOnly
from .xoauth2 import XOAUTH2TokenAcquirer
from .xoauth2 import XOAUTH2MailerFactory
credentialsStore = CredentialsStore.create(applicationName)
oauth2authenticator = OAuth2AuthenticatorRefreshingOnly(
credentialsStore,
)
tokenAcquirer = XOAUTH2TokenAcquirer(
oauth2authenticator=oauth2authenticator,
)
mailerFactory = XOAUTH2MailerFactory(
reactor,
remoteHost,
remotePort,
tokenAcquirer,
None,
)
smtpDeliveryFactory = MessageDeliveryFactory(mailerFactory=mailerFactory)
smtpServerProtoFactory = SMTPServerProtocolFactory(
deliveryFactory=smtpDeliveryFactory,
)
smtpServer = TCPServer(
listenPort,
smtpServerProtoFactory,
interface=bindAddress
)
smtpServer.setServiceParent(application)
return application
def setupReceivers(root_service, settings):
from carbon.protocols import MetricLineReceiver, MetricPickleReceiver, MetricDatagramReceiver
for protocol, interface, port in [
(MetricLineReceiver, settings.LINE_RECEIVER_INTERFACE,
settings.LINE_RECEIVER_PORT),
(MetricPickleReceiver, settings.PICKLE_RECEIVER_INTERFACE,
settings.PICKLE_RECEIVER_PORT)
]:
if port:
factory = ServerFactory()
factory.protocol = protocol
service = TCPServer(port, factory, interface=interface)
service.setServiceParent(root_service)
if settings.ENABLE_UDP_LISTENER:
service = UDPServer(int(settings.UDP_RECEIVER_PORT),
MetricDatagramReceiver(),
interface=settings.UDP_RECEIVER_INTERFACE)
service.setServiceParent(root_service)
if settings.ENABLE_AMQP:
from carbon import amqp_listener
amqp_host = settings.AMQP_HOST
amqp_port = settings.AMQP_PORT
amqp_user = settings.AMQP_USER
amqp_password = settings.AMQP_PASSWORD
amqp_verbose = settings.AMQP_VERBOSE
amqp_vhost = settings.AMQP_VHOST
amqp_spec = settings.AMQP_SPEC
amqp_exchange_name = settings.AMQP_EXCHANGE
factory = amqp_listener.createAMQPListener(
amqp_user,
amqp_password,
vhost=amqp_vhost,
spec=amqp_spec,
exchange_name=amqp_exchange_name,
verbose=amqp_verbose)
service = TCPClient(amqp_host, amqp_port, factory)
service.setServiceParent(root_service)
if settings.ENABLE_MANHOLE:
from carbon import manhole
factory = manhole.createManholeListener()
service = TCPServer(settings.MANHOLE_PORT,
factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
def makeService(self, options):
config = options
import cap
from cap_twisted import service as mainrpc
import sys
s = MultiService()
sys.path.insert(1, cap.__path__[0])
del sys.modules["cap"]
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cap.settings")
mysql_url = options["mysql_url"].strip()
try:
a, b = mysql_url.split(":")
mysql_host = a
mysql_port, mysql_db = b.split("/")
mysql_port = int(mysql_port)
except:
print "mysql相关配置错误"
raise Exception("mysql相关配置错误")
else:
mysql_user = options["mysql_user"]
mysql_password = options["mysql_password"]
os.config = [
mysql_host, mysql_port, mysql_db, mysql_user, mysql_password
]
os.work_dir = options["work_dir"]
os.host = options["host"]
from django.core.handlers.wsgi import WSGIHandler
application = WSGIHandler()
resource = WSGIResource(reactor, reactor.getThreadPool(), application)
ui_service = TCPServer(9912,
server.Site(resource),
interface=config["host"])
serverfactory = server.Site(
mainrpc.MainRpc(config["host"], config["work_dir"]))
slave_service = TCPServer(9913,
serverfactory,
interface=config["host"])
slave_service.setServiceParent(s)
ui_service.setServiceParent(s)
return s
def setupReceivers(root_service, settings):
from carbon.protocols import MetricLineReceiver, MetricPickleReceiver, MetricDatagramReceiver
for protocol, interface, port in [
(MetricLineReceiver, settings.LINE_RECEIVER_INTERFACE, settings.LINE_RECEIVER_PORT),
(MetricPickleReceiver, settings.PICKLE_RECEIVER_INTERFACE, settings.PICKLE_RECEIVER_PORT)
]:
if port:
factory = ServerFactory()
factory.protocol = protocol
service = TCPServer(port, factory, interface=interface)
service.setServiceParent(root_service)
if settings.ENABLE_UDP_LISTENER:
service = UDPServer(int(settings.UDP_RECEIVER_PORT),
MetricDatagramReceiver(),
interface=settings.UDP_RECEIVER_INTERFACE)
service.setServiceParent(root_service)
if settings.ENABLE_AMQP:
from carbon import amqp_listener
amqp_host = settings.AMQP_HOST
amqp_port = settings.AMQP_PORT
amqp_user = settings.AMQP_USER
amqp_password = settings.AMQP_PASSWORD
amqp_verbose = settings.AMQP_VERBOSE
amqp_vhost = settings.AMQP_VHOST
amqp_spec = settings.AMQP_SPEC
amqp_exchange_name = settings.AMQP_EXCHANGE
factory = amqp_listener.createAMQPListener(
amqp_user,
amqp_password,
vhost=amqp_vhost,
spec=amqp_spec,
exchange_name=amqp_exchange_name,
verbose=amqp_verbose)
service = TCPClient(amqp_host, amqp_port, factory)
service.setServiceParent(root_service)
if settings.ENABLE_MANHOLE:
from carbon import manhole
factory = manhole.createManholeListener()
service = TCPServer(
settings.MANHOLE_PORT,
factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
schedpath = config.get('scheduler', 'scrapyd.scheduler.SpiderScheduler')
schedCls = load_object(schedpath)
scheduler = schedCls(config, app)
environment = Environment(config)
pubsub_path = config.get('pubsub', 'scrapyd.pubsub.BasePubSub')
pubsubCls = load_object(pubsub_path)
pubsub = pubsubCls(config, app)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
app.setComponent(IPubSub, pubsub)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config, app)
taskpoller = TaskPoller(config, app) #
app.setComponent(IPoller, poller)
app.setComponent(IPoller, taskpoller) #
timer = TimerService(poll_interval, poller.poll)
tasktimer = TimerService(poll_interval, taskpoller.poll) #
timer.setServiceParent(app)
tasktimer.setServiceParent(app) #
http_port = config.getint('http_port', 9090)
bind_address = config.get('bind_address', '127.0.0.1')
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
webservice.setServiceParent(app)
return app
def createCacheService(config):
from carbon.cache import MetricCache
from carbon.conf import settings
from carbon.events import metricReceived
from carbon.protocols import protocolManager
from carbon.protocols import CacheQueryHandler
# Configure application components
metricReceived.installHandler(MetricCache.store)
root_service = createBaseService(config)
factory = protocolManager.createFactory(CacheQueryHandler)
service = TCPServer(int(settings.CACHE_QUERY_PORT), factory,
interface=settings.CACHE_QUERY_INTERFACE)
service.setServiceParent(root_service)
# have to import this *after* settings are defined
from carbon.writer import WriterService
service = WriterService()
service.setServiceParent(root_service)
return root_service
def application(config):
app = Application("Scrapyd")
http_port = config.getint("http_port", 6800)
bind_address = config.get("bind_address", "0.0.0.0")
poll_interval = config.getfloat("poll_interval", 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get("launcher", "scrapyd.launcher.Launcher")
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get("webroot", "scrapyd.website.Root")
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port, server.Site(webcls(config, app)), interface=bind_address)
log.msg(
format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port,
)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
log.msg("Scrapyd web console available at http://localhost:%s/" % http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application():
app = Application("Scrapyd")
config = Config()
http_port = config.getint('http_port', 6800)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
class Trigger(Resource):
def __init__(self,password):
self.password = open(password,'r').read().strip()
def render_POST(self, request):
if self.password == request.content.read().decode('utf-8'):
print('success')
GPIO.output(GARAGE,False)
time.sleep(1)
GPIO.output(GARAGE,True)
else:
print('fail')
return b'<html></html>'
try:
GPIO.cleanup()
GPIO.setmode(GPIO.BOARD)
GPIO.setup(GARAGE,GPIO.OUT,initial=True)
except Exception as e:
print(e, 'Failed to setup GPIO')
exit(0)
garage = Resource()
garage.putChild(b"trigger",Trigger(PASSWD_PATH))
root = Resource()
root.putChild(b"garage", garage)
factory = Site(root)
application = Application("gdo")
server = TCPServer(8001, factory)
server.setServiceParent(application)
def _listen(self):
"Set service to listen on tcp port"
masterService = TCPServer(self._masterPort, self._taskmaster)
masterService.setServiceParent(self)
log.msg("TaskMaster is listening on port: %d" % self._masterPort)
def createBaseService(config):
from carbon.conf import settings
from carbon.protocols import (MetricLineReceiver, MetricPickleReceiver,
MetricDatagramReceiver)
root_service = CarbonRootService()
root_service.setName(settings.program)
use_amqp = settings.get("ENABLE_AMQP", False)
if use_amqp:
from carbon import amqp_listener
amqp_host = settings.get("AMQP_HOST", "localhost")
amqp_port = settings.get("AMQP_PORT", 5672)
amqp_user = settings.get("AMQP_USER", "guest")
amqp_password = settings.get("AMQP_PASSWORD", "guest")
amqp_verbose = settings.get("AMQP_VERBOSE", False)
amqp_vhost = settings.get("AMQP_VHOST", "/")
amqp_spec = settings.get("AMQP_SPEC", None)
amqp_exchange_name = settings.get("AMQP_EXCHANGE", "graphite")
for interface, port, backlog, protocol in (
(settings.LINE_RECEIVER_INTERFACE, settings.LINE_RECEIVER_PORT,
settings.LINE_RECEIVER_BACKLOG, MetricLineReceiver),
(settings.PICKLE_RECEIVER_INTERFACE, settings.PICKLE_RECEIVER_PORT,
settings.PICKLE_RECEIVER_BACKLOG, MetricPickleReceiver)):
if port:
factory = ServerFactory()
factory.protocol = protocol
service = TCPServer(int(port),
factory,
interface=interface,
backlog=backlog)
service.setServiceParent(root_service)
if settings.ENABLE_UDP_LISTENER:
service = UDPServer(int(settings.UDP_RECEIVER_PORT),
MetricDatagramReceiver(),
interface=settings.UDP_RECEIVER_INTERFACE)
service.setServiceParent(root_service)
if use_amqp:
factory = amqp_listener.createAMQPListener(
amqp_user,
amqp_password,
vhost=amqp_vhost,
spec=amqp_spec,
exchange_name=amqp_exchange_name,
verbose=amqp_verbose)
service = TCPClient(amqp_host, int(amqp_port), factory)
service.setServiceParent(root_service)
if settings.ENABLE_MANHOLE:
from carbon import manhole
factory = manhole.createManholeListener()
service = TCPServer(int(settings.MANHOLE_PORT),
factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
if settings.USE_WHITELIST:
from carbon.regexlist import WhiteList, BlackList
WhiteList.read_from(settings["whitelist"])
BlackList.read_from(settings["blacklist"])
# Instantiate an instrumentation service that will record metrics about
# this service.
from carbon.instrumentation import InstrumentationService
service = InstrumentationService()
service.setServiceParent(root_service)
return root_service
def createBaseService(config):
from carbon.conf import settings
from carbon.protocols import (MetricLineReceiver, MetricPickleReceiver,
MetricDatagramReceiver, CmdLineReceiver)
root_service = CarbonRootService()
root_service.setName(settings.program)
use_amqp = settings.get("ENABLE_AMQP", False)
if use_amqp:
from carbon import amqp_listener
amqp_host = settings.get("AMQP_HOST", "localhost")
amqp_port = settings.get("AMQP_PORT", 5672)
amqp_user = settings.get("AMQP_USER", "guest")
amqp_password = settings.get("AMQP_PASSWORD", "guest")
amqp_verbose = settings.get("AMQP_VERBOSE", False)
amqp_vhost = settings.get("AMQP_VHOST", "/")
amqp_spec = settings.get("AMQP_SPEC", None)
amqp_exchange_name = settings.get("AMQP_EXCHANGE", "graphite")
for interface, port, protocol in ((settings.LINE_RECEIVER_INTERFACE,
settings.LINE_RECEIVER_PORT,
MetricLineReceiver),
(settings.PICKLE_RECEIVER_INTERFACE,
settings.PICKLE_RECEIVER_PORT,
MetricPickleReceiver),
(settings.CMD_LINE_RECEIVER_INTERFACE,
settings.CMD_LINE_RECEIVER_PORT,
CmdLineReceiver)):
if port:
factory = ServerFactory()
factory.protocol = protocol
service = TCPServer(int(port), factory, interface=interface)
service.setServiceParent(root_service)
if settings.ENABLE_UDP_LISTENER:
service = UDPServer(int(settings.UDP_RECEIVER_PORT),
MetricDatagramReceiver(),
interface=settings.UDP_RECEIVER_INTERFACE)
service.setServiceParent(root_service)
if use_amqp:
factory = amqp_listener.createAMQPListener(
amqp_user, amqp_password,
vhost=amqp_vhost, spec=amqp_spec,
exchange_name=amqp_exchange_name,
verbose=amqp_verbose)
service = TCPClient(amqp_host, int(amqp_port), factory)
service.setServiceParent(root_service)
if settings.ENABLE_MANHOLE:
from carbon import manhole
factory = manhole.createManholeListener()
service = TCPServer(int(settings.MANHOLE_PORT), factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
if settings.USE_WHITELIST:
from carbon.regexlist import WhiteList,BlackList
WhiteList.read_from(settings["whitelist"])
BlackList.read_from(settings["blacklist"])
# Instantiate an instrumentation service that will record metrics about
# this service.
from carbon.instrumentation import InstrumentationService
service = InstrumentationService()
service.setServiceParent(root_service)
return root_service
#------------------------------------------------------------------------------#
import settings
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.engine.url import URL
from starpy import fastagi
import agi
#start logging
logFile = logfile.LogFile.fromFullPath('/var/log/qmlremis/agi.log')
log.addObserver(log.FileLogObserver(logFile).emit)
#connect mysql
engine = create_engine(URL(**settings.DATABASE))
Session = sessionmaker(bind=engine)
session = Session()
#connect websocket
ws = OperClientFactory("ws://localhost:9000/Oper")
connectWS(ws)
ragi = agi.RemisAgi(session, ws.protocol())
factory = fastagi.FastAGIFactory(ragi.main)
log.msg("start QmlRemis AGI Server")
application = Application("QmlRemis AGI Server")
srv = TCPServer(4573, factory)
srv.setServiceParent(application)
def connectWWW(self, svc, loginPortal, opts):
site = self.buildSite(svc, loginPortal, opts)
www = TCPServer(opts["www-port"], site)
www.setServiceParent(svc)
return www
class VirginCoinServer(pb.Root):
addresses = ('ad1', 2, 'ad2', 3)
def __init__(self):
# Fire callback when server is ready
self.on_load = defer.Deferred()
for method in self._remote_methods:
method = str(method).strip()
VirginCoinServer.__dict__['remote_' + method] = VirginCoinServer.__dict__[method]
self._reconnect_state = {'status': 'none'}
self._shutdown_in_progress = False
_remote_methods = ('set_coinbase', 'set_some')
#@staticmethod
def set_some(self, slave):
print "received a slave called", slave
slave.callRemote("print", 22)
slave.callRemote("set_addresses", self.addresses)
def set_coinbase(self):
pass
serverfactory = pb.PBServerFactory(VirginCoinServer())
application = Application("virgin")
VServerService = TCPServer(3305, serverfactory)
VServerService.setServiceParent(application)
isLeaf = True
def __init__(self):
Resource.__init__(self)
def render_GET(self, request):
deferred = flatten(request, WebPage(), request.write)
deferred.addCallback(finish_request, request)
# in case of error we don't want the browser to stay stuck
deferred.addErrback(flattenerror, request)
return NOT_DONE_YET
def render_POST(self, request):
baseurl = request.args["baseurl"][0]
revurl = reverseUrl(baseurl)
row = getRow(revurl)
deferred = flatten(request, WebPage(baseurl, revurl, row), request.write)
deferred.addCallback(finish_request, request)
deferred.addErrback(flattenerror, request)
return NOT_DONE_YET
webservice = TCPServer(config.HTTP_PORT, server.Site(Root()), interface=config.BIND_ADDRESS)
log.msg(
format="Web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=config.BIND_ADDRESS,
http_port=config.HTTP_PORT,
)
application = Application("MiniFrontend")
webservice.setServiceParent(application)
