def __init__(self, **options):
MultiService.__init__(self)
address = ('gpcm.gamespy.com', 29900)
sFact = MowLoginServerFactory()
#sFact = makeTCPFwdFactory('gamespy.gpcmCli', 'gamespy.gpcmSrv', makeRecv(ProxyClient), makeRecv(ProxyServer))(*address)
self.addService(TCPServer(address[1], sFact))
address = ('peerchat.gamespy.com', 6667)
sFact = ProxyPeerchatServerFactory(gameId, *address)
#sFact = makeTCPFwdFactory('gamespy.peerCli', 'gamespy.peerSrv', makeRecv(ProxyClient, True), makeRecv(ProxyServer, True))(*address)
self.addService(TCPServer(address[1], sFact))
address = ('%s.gamestats.gamespy.com' % (gameId, ), 29920)
sFact = makeTCPFwdFactory('gamespy.stat1cli', 'gamespy.stat1srv',
makeRecv(ProxyClient, True),
makeRecv(ProxyServer, True))(*address)
self.addService(TCPServer(address[1], sFact))
# ('menofwarpc.master.gamespy.com', 29910), # 29910 UDP is keycheck ('gamespy' xor), 27900 for gameinfo/natneg(?)
address = ('%s.master.gamespy.com' % (gameId, ), 28910)
sFact = makeTCPFwdFactory('gamespy.masterCli', 'gamespy.masterSrv',
makeRecv(ProxyClient, True),
makeRecv(ProxyServer, True))(*address)
#sFact = ProxyMasterServerFactory(gameId, *address)
self.addService(TCPServer(address[1], sFact))
def setupManhole(application, config):
"""Setup an SSH manhole for the API service.
The manhole port is taken from the C{manhole-port} option in the config
file. If this option is not provided the api port plus 100 is used.
@param application: The fluidinfo API L{Application} object.
@param config: The configuration object.
"""
servicePort = config.getint('service', 'port')
if config.has_option('service', 'manhole-port'):
manholePort = config.getint('service', 'manhole-port')
else:
manholePort = servicePort + 100
def getManhole(_):
manhole = Manhole(globals())
ps1 = 'fluidinfo-api [%d] > ' % servicePort
ps2 = '... '.rjust(len(ps1))
manhole.ps = (ps1, ps2)
return manhole
realm = TerminalRealm()
realm.chainedProtocolFactory.protocolFactory = getManhole
portal = Portal(realm)
portal.registerChecker(SSHPublicKeyDatabase())
factory = ConchFactory(portal)
manholeService = TCPServer(manholePort, factory, interface='127.0.0.1')
manholeService.setServiceParent(application)
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg("Scrapyd web console available at http://%s:%s/" %
(bind_address, http_port))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def get_application(config):
app = Application('Scrapyd')
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd_mongodb.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(
http_port, server.Site(Root(config, app)),
interface=bind_address)
log.msg('http://%(bind_address)s:%(http_port)s/' % {'bind_address':bind_address, 'http_port':http_port})
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
root = Root(config, app)
root = configRoot(root, config)
webservice = TCPServer(http_port, server.Site(root))
log.msg("Scrapyd web console available at http://localhost:%s/" % http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
portal = Portal(PublicHTMLRealm(config, app), [FilePasswordDB(str(config.get('passwd', '')))])
credentialFactory = DigestCredentialFactory("md5", "Go away")
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(HTTPAuthSessionWrapper(portal, [credentialFactory])))
log.msg("Scrapyd web console available at http://localhost:%s/" % http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = int(environ.get('PORT', config.getint('http_port', 6800)))
config.cp.set('scrapyd', 'database_url', environ.get('DATABASE_URL'))
poller = Psycopg2QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = Psycopg2SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
log.msg("Scrapyd web console available at http://localhost:%s/ (HEROKU)" %
http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg("Scrapyd web console available at http://%s:%s/" % (bind_address, http_port))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def build(cls, root_service):
if not settings.ENABLE_MANHOLE:
return
factory = createManholeListener()
service = TCPServer(settings.MANHOLE_PORT, factory, interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
def run_interface(config):
factory = ServerFactory()
factory.protocol = lambda: TelnetTransport(JunctionTelnetInterface)
port = int(config['App']['TelnetInterfacePort'])
logging.info("starting telnet cli interface on port %d" % port)
service = TCPServer(port, factory)
service.startService()
def get_application(arguments):
ServiceRoot = load_object(settings.SERVICE_ROOT)
site = Site(ServiceRoot())
application = Application('scrapyrt')
server = TCPServer(arguments.port, site, interface=arguments.ip)
server.setServiceParent(application)
return application
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def get_application(arguments):
ServiceRoot = load_object(settings.SERVICE_ROOT)
site = Site(ServiceRoot())
application = Application('scrapyrt')
server = TCPServer(arguments.port, site, interface=arguments.ip)
server.setServiceParent(application)
return application
def application(config):
app = Application("Scrapyd")
http_port = int(environ.get('PORT', config.getint('http_port', 6800)))
config.cp.set('scrapyd', 'database_url', environ.get('DATABASE_URL'))
poller = Psycopg2QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = Psycopg2SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
log.msg("Scrapyd web console available at http://localhost:%s/ (HEROKU)"
% http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def makeService(self, options):
"""
Create a service which will run a scram server.
@param options: mapping of configuration
"""
from scram.network import ScramFactory
from scram.world import (TCP_SERVICE_NAME, SCRAM_SERVICE_NAME, ScramService, World)
from twisted.internet import reactor
from twisted.application.service import MultiService
from twisted.protocols.policies import TrafficLoggingFactory
world = World(granularity=60, platformClock=reactor)
service = MultiService()
factory = ScramFactory(world)
if options['log-directory'] is not None:
factory = TrafficLoggingFactory(
factory, join(options['log-directory'], 'scram'))
tcp = TCPServer(options['port'], factory, interface='127.0.0.1')
tcp.setName(TCP_SERVICE_NAME)
tcp.setServiceParent(service)
scram = ScramService(world)
scram.setName(SCRAM_SERVICE_NAME)
scram.setServiceParent(service)
return service
def application(config, components=interfaces):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
for interface, key in interfaces:
path = config.get(key)
cls = load_object(path)
component = cls(config)
app.setComponent(interface, component)
poller = component
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
poll_every = config.getint("poll_every", 5)
timer = TimerService(poll_every, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def __init__(self, dbpool, services):
self.dbpool = dbpool
self.services = services
self.isRunning = False
port = services["config"]["flash-policy-port"]
factory = FlashPolicyFactory(services["config"])
TCPServer.__init__(self, port, factory)
def registry(self):
root = NestRoot(self.app, debug=False)
for (path, service) in self.webservices.items():
root.putChild(path.encode('utf-8'), service(root))
server = TCPServer(self.port, Site(root), interface=self.host)
server.setServiceParent(self.app)
return self.app
def __init__(self, env):
self.env = env
http_port = env.config.getint('web', 'http_port') or HTTP_PORT
log_file = env.config.get('web', 'http_log_file') or None
if not os.path.isabs(log_file):
log_file = os.path.join(self.env.config.path, log_file)
factory = WebServiceFactory(env, log_file)
TCPServer.__init__(self, http_port, factory)
def makeService(self, options):
config = options
s = MultiService()
from cap_twisted import service as mainrpc
serverfactory = server.Site(mainrpc.MainRpc(config["master"],config["work_dir"]))
slave_service = TCPServer(9913, serverfactory, interface=config["host"])
slave_service.setServiceParent(s)
return s
def makeService(self, options):
"""
Construct the mapiphany
"""
from mapiphany.webserver import WebSite
site = WebSite()
ws = TCPServer(int(options['port']), site)
ws.site = site
return ws
def makeService(self, opts):
from carchive._conf import ConfigDict
from carchive.archmiddle.proxy import buildResource
from carchive.archmiddle.info import InfoCache, KeyNameMap
server = ConfigDict(opts['config'], 'server')
mapping = ConfigDict(opts['config'], 'mapping')
handle = Log2Twisted()
handle.setFormatter(logging.Formatter("%(message)s"))
root = logging.getLogger()
root.addHandler(handle)
root.setLevel(server.getint('log.level', logging.INFO))
# turn down the noise level...
CRL = logging.getLogger('carchive.rpcmunge')
if CRL.isEnabledFor(logging.DEBUG):
CRL.setLevel(logging.INFO)
mservice = service.MultiService()
_M = []
for k, v in mapping.iteritems():
v = v.split()
_M.append((k, int(v[0]), v[1:]))
KM = KeyNameMap(_M)
info = InfoCache(server['url'], KM)
root, leaf = buildResource(info, reactor)
fact = Site(root)
info.pvlimit = server.getint('cache.limit', 500)
info.timeout = server.getfloat('cache.timeout', 3600)
mservice.addService(
TCPServer(server.getint('port'),
fact,
interface=server.get('interface', '')))
if ShellFactory and server.getint('manhole.port', 0):
print 'Opening Manhole'
SF = ShellFactory()
SS = TCPServer(server.getint('manhole.port', 0),
SF,
interface='127.0.0.1')
# populate manhole shell locals
SF.namespace['site'] = fact
SF.namespace['node'] = leaf
SF.namespace['info'] = info
mservice.addService(SS)
else:
print 'No Manhole'
return mservice
def build(cls, root_service):
if not settings.ENABLE_MANHOLE:
return
factory = createManholeListener()
service = TCPServer(settings.MANHOLE_PORT,
factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
def makeService(config):
event_db = Event_DB(config['eventdb'])
LoopingCall(event_db.prune, MAX_AGE).start(PRUNE_INTERVAL)
broker_service = MultiService()
if config['broadcast']:
broadcaster_factory = VOEventBroadcasterFactory(
config["local-ivo"], config['broadcast-test-interval']
)
if log.LEVEL >= log.Levels.INFO: broadcaster_factory.noisy = False
broadcaster_service = TCPServer(
config['broadcast-port'],
broadcaster_factory
)
broadcaster_service.setName("Broadcaster")
broadcaster_service.setServiceParent(broker_service)
# If we're running a broadcast, we will rebroadcast any events we
# receive to it.
config['handlers'].append(EventRelay(broadcaster_factory))
if config['receive']:
receiver_factory = VOEventReceiverFactory(
local_ivo=config['local-ivo'],
validators=[
CheckPreviouslySeen(event_db),
CheckSchema(
os.path.join(comet.__path__[0], "schema/VOEvent-v2.0.xsd")
),
CheckIVORN()
],
handlers=config['handlers']
)
if log.LEVEL >= log.Levels.INFO: receiver_factory.noisy = False
whitelisting_factory = WhitelistingFactory(receiver_factory, config['whitelist'])
if log.LEVEL >= log.Levels.INFO: whitelisting_factory.noisy = False
receiver_service = TCPServer(config['receive-port'], whitelisting_factory)
receiver_service.setName("Receiver")
receiver_service.setServiceParent(broker_service)
for host, port in config["remotes"]:
subscriber_factory = VOEventSubscriberFactory(
local_ivo=config["local-ivo"],
validators=[CheckPreviouslySeen(event_db)],
handlers=config['handlers'],
filters=config['filters']
)
if log.LEVEL >= log.Levels.INFO: subscriber_factory.noisy = False
remote_service = TCPClient(host, port, subscriber_factory)
remote_service.setName("Remote %s:%d" % (host, port))
remote_service.setServiceParent(broker_service)
if not broker_service.services:
reactor.callWhenRunning(log.warning, "No services requested; stopping.")
reactor.callWhenRunning(reactor.stop)
return broker_service
def _makeSiteService(self, papi_xmlrpc, config):
"""Create the site service."""
site_root = Resource()
site_root.putChild("api", papi_xmlrpc)
site = Site(site_root)
site_port = config["port"]
site_interface = config["interface"]
site_service = TCPServer(site_port, site, interface=site_interface)
site_service.setName("site")
return site_service
def _makeSiteService(self, papi_xmlrpc, config):
"""Create the site service."""
site_root = Resource()
site_root.putChild("api", papi_xmlrpc)
site = Site(site_root)
site_port = config["port"]
site_interface = config["interface"]
site_service = TCPServer(site_port, site, interface=site_interface)
site_service.setName("site")
return site_service
def configure_services(self, configuration):
read_configuration()
for section in configuration.sections():
if section.startswith("world "):
factory = BravoFactory(section[6:])
server = TCPServer(factory.port, factory,
interface=factory.interface)
server.setName(factory.name)
self.addService(server)
elif section.startswith("irc "):
try:
from bravo.irc import BravoIRC
except ImportError:
log.msg("Couldn't import IRC stuff!")
else:
factory = BravoIRC(self.namedServices, section[4:])
client = TCPClient(factory.host, factory.port, factory)
client.setName(factory.name)
self.addService()
elif section.startswith("infiniproxy "):
factory = BetaProxyFactory(section[12:])
server = TCPServer(factory.port, factory)
server.setName(factory.name)
self.addService(server)
elif section.startswith("infininode "):
factory = InfiniNodeFactory(section[11:])
server = TCPServer(factory.port, factory)
server.setName(factory.name)
self.addService(server)
def init(application):
global PORT, MATRIX_REMOTE
root = ApplicationHome()
root.putChild("rooms", RoomsPage())
root.putChild("users", UsersPage())
root.putChild("transactions", TransactionsPage())
factory = Site(root)
j = TCPServer(PORT, factory)
j.setServiceParent(application)
if MATRIX_REMOTE is not None:
startTunnelToMatrix()
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(webcls(config, app)),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
host = get_host_ip(config)
redis_host = config.get('redis_host', 'localhost')
redis_port = config.get('redis_port', 6379)
redis_db = config.get('redis_db', 0)
redis_pool = redis.ConnectionPool(host=redis_host,
port=redis_port,
db=redis_db)
register_to_redis(config, redis_pool)
log.msg('Registering scrapyd [{}] to redis {}:{} at db {}'.format(
host, redis_host, redis_port, redis_db))
# log.msg('2018-11-03 10:10 am')
redis_interval = config.getfloat('redis_interval', 5)
register_timer = TimerService(redis_interval, register_to_redis, config,
redis_pool)
register_timer.setServiceParent(app)
return app
def configure_service(self):
"""
Instantiation and startup for the RESTful API.
"""
root = APIResource()
factory = Site(root)
port = settings.API_PORT
server = TCPServer(port, factory)
server.setName("WebAPI")
self.addService(server)
def makeService(self, options):
top_service = service.MultiService()
server_service = ServerService(options["map"], int(options["numplayers"]))
server_service.setServiceParent(top_service)
server_factory = MyServerFactory(server_service)
server = TCPServer(int(options["port"]), server_factory)
server.setServiceParent(top_service)
return top_service
def create_status_service(storage, parent_service, port, user_id=0, ssl_context_factory=None):
"""Create the status service."""
root = resource.Resource()
root.putChild("status", _Status(storage, user_id))
root.putChild("+meliae", MeliaeResource())
root.putChild("+gc-stats", GCResource())
site = server.Site(root)
if ssl_context_factory is None:
service = TCPServer(port, site)
else:
service = SSLServer(port, site, ssl_context_factory)
service.setServiceParent(parent_service)
return service
def addProtocol(self, protocol, port):
self.log.info(
"Setting service for protocol {protocol!r} on port {port}...",
protocol=protocol, port=port,
)
# TCP Service
tcpFactory = SpawningInheritingProtocolFactory(
self.dispatcher, self.spawningService, protocol
)
tcpService = TCPServer(port, tcpFactory)
tcpService.setServiceParent(self)
def __init__(self, dbpool, services, reactor=None):
## lazy import to avoid reactor install upon module import
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
self.dbpool = dbpool
self.services = services
self.isRunning = False
port = services["config"]["flash-policy-port"]
allowedPort = services["config"]["hub-websocket-port"]
factory = FlashPolicyFactory(allowedPort, reactor)
TCPServer.__init__(self, port, factory)
def __init__(self, dbpool, services, reactor = None):
## lazy import to avoid reactor install upon module import
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
self.dbpool = dbpool
self.services = services
self.isRunning = False
port = services["config"]["flash-policy-port"]
allowedPort = services["config"]["hub-websocket-port"]
factory = FlashPolicyFactory(allowedPort, reactor)
TCPServer.__init__(self, port, factory)
def create_status_service(storage, parent_service, port,
user_id=0, ssl_context_factory=None):
"""Create the status service."""
root = resource.Resource()
root.putChild('status', _Status(storage, user_id))
root.putChild('+meliae', MeliaeResource())
root.putChild('+gc-stats', GCResource())
site = server.Site(root)
if ssl_context_factory is None:
service = TCPServer(port, site)
else:
service = SSLServer(port, site, ssl_context_factory)
service.setServiceParent(parent_service)
return service
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
username = config.get('username', '')
password = config.get('password', '')
if username and password:
if ':' in username:
sys.exit("The `username` option contains illegal character ':', "
"check and update the configuration file of Scrapyd")
portal = Portal(PublicHTMLRealm(webcls(config, app)),
[StringCredentialsChecker(username, password)])
credential_factory = BasicCredentialFactory("Auth")
resource = HTTPAuthSessionWrapper(portal, [credential_factory])
log.msg("Basic authentication enabled")
else:
resource = webcls(config, app)
log.msg("Basic authentication disabled as either `username` or `password` is unset")
webservice = TCPServer(http_port, server.Site(resource), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def makeService(self, opts):
from carchive.a2aproxy.resource import buildResource
from carchive.util import LimitedSite, LimitedTCPServer
L = logging.INFO
if opts['debug']:
L = logging.DEBUG
handle = Log2Twisted()
handle.setFormatter(
logging.Formatter("%(levelname)s:%(name)s %(message)s"))
root = logging.getLogger()
root.addHandler(handle)
root.setLevel(L)
serv = service.MultiService()
fact = LimitedSite(buildResource(opts['appl']))
serv.addService(
LimitedTCPServer(opts['port'], fact, interface=opts['ip']))
if ShellFactory and opts['manhole']:
print 'Opening Manhole'
SF = ShellFactory()
SS = TCPServer(opts['manhole'], SF, interface='127.0.0.1')
# populate manhole shell locals
SF.namespace['site'] = fact
serv.addService(SS)
else:
print 'No Manhole'
return serv
def __init__(self, **options):
MultiService.__init__(self)
self.addService(
TCPServer(
options['port'],
PeerchatProxyServerFactory(options['game'], options['host'],
6667)))
def create_status_service(proxy_server, port):
"""Create the status service."""
root = resource.Resource()
root.putChild('status', _Status(proxy_server))
site = server.Site(root)
service = TCPServer(port, site)
return service
def makeService(self, options):
config = options
import cap
import sys
sys.path.insert(1,cap.__path__[0])
del sys.modules["cap"]
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cap.settings")
mysql_url = options["mysql_url"].strip()
try:
a, b = mysql_url.split(":")
mysql_host = a
mysql_port, mysql_db = b.split("/")
mysql_port = int(mysql_port)
except:
print "mysql相关配置错误"
raise Exception("mysql相关配置错误")
else:
mysql_user = options["mysql_user"]
mysql_password = options["mysql_password"]
os.config = [mysql_host,mysql_port,mysql_db,mysql_user,mysql_password]
from django.core.handlers.wsgi import WSGIHandler
application = WSGIHandler()
resource = WSGIResource(reactor, reactor.getThreadPool(), application)
ui_service=TCPServer(9912,server.Site(resource),interface=config["host"])
return ui_service
def makeService(self, options):
port, site, application = setupApplication(options)
# Check database health.
logging.info('Checking database health.')
try:
verifyStore()
except Exception as error:
logging.error(error)
logging.critical('Shutting down.')
sys.exit(1)
else:
logging.info('Database is up-to-date.')
# Log configuration parameters.
config = getConfig()
logging.info('PID is %s', os.getpid())
logging.info('service/temp-path is %s',
config.get('service', 'temp-path'))
logging.info('service/max-threads is %s',
config.get('service', 'max-threads'))
logging.info('service/port is %s', config.get('service', 'port'))
logging.info('store/main-uri is %s', config.get('store', 'main-uri'))
logging.info('index/url is %s', config.get('index', 'url'))
# Register the application.
return TCPServer(int(options['port']), site)
def createBaseService(options):
root_service = RurouniRootService()
root_service.setName('rurouni')
receive_services = (
(settings.LINE_RECEIVER_INTERFACE,
settings.LINE_RECEIVER_PORT,
protocols.MetricLineReceiver
),
(settings.PICKLE_RECEIVER_INTERFACE,
settings.PICKLE_RECEIVER_PORT,
protocols.MetricPickleReceiver
),
)
for interface, port, protocol in receive_services:
if port:
factory = ServerFactory()
factory.protocol = protocol
service = TCPServer(int(port), factory, interface=interface)
service.setServiceParent(root_service)
from rurouni.state.instrumentation import InstrumentationService
service = InstrumentationService()
service.setServiceParent(root_service)
return root_service
def _configure_web_server(self, config, controller):
interfaces, https, key_file, cert_file, chain_file, _, _ = \
self._validate_web_config(config)
site = server.Site(get_web_app(config, controller))
web_servers = []
for interface, port in interfaces:
if https:
cf = SSLCertOptions(key_file, cert_file, chain_file)
web_server = SSLServer(port, site, cf, interface=interface)
method = 'https'
else:
web_server = TCPServer(port, site, interface=interface)
method = 'http'
web_servers.append(web_server)
if ':' in interface:
interface = '[{}]'.format(interface)
log.msg(format="Scrapy-Do web interface is available at "
"%(method)s://%(interface)s:%(port)s/",
method=method,
interface=interface,
port=port)
return web_servers
def application(config):
app = Application("Flowder")
app_id = config.get('app_id', 'fw0')
logfile = config.get('logfile', '/var/log/flowder.log')
loglevel = config.get('loglevel', 'info')
db_file = config.get('db_file', 'flowder')
rest_port = config.getint('rest_port', 4000)
rest_bind = config.get('rest_bind', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 1)
poll_size = config.getint("poll_size", 5)
signalmanager = SignalManager()
app.setComponent(ISignalManager, signalmanager)
fetcher = FetcherService(config)
fetcher.setServiceParent(app)
poller = QueuePoller(app, poll_size)
poller.setServiceParent(app)
db_file = '%s.db' % db_file
task_storage = FileDownloaderTaskStorage(app, db_file)
task_storage.setServiceParent(app)
timer = TimerService(poll_interval, poller.poll)
timer.setServiceParent(app)
scheduler = TaskScheduler(config, app)
scheduler.setServiceParent(app)
laupath = config.get('launcher', 'flowder.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(app, config)
launcher.setServiceParent(app)
restService = TCPServer(rest_port,
server.Site(Root(app, config)),
interface=rest_bind)
restService.setServiceParent(app)
amqp_publisher = AmqpService(app, config)
amqp_publisher.setServiceParent(app)
log.msg("Starting Flowder services (;-)")
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
jspath = config.get('jobstorage', 'scrapyd.jobstorage.MemoryJobStorage')
jscls = load_object(jspath)
jobstorage = jscls(config)
app.setComponent(IJobStorage, jobstorage)
eggstorage = config.get('eggstorage',
'scrapyd.eggstorage.FilesystemEggStorage')
eggstoragecls = load_object(eggstorage)
app.setComponent(IEggStorage, eggstoragecls(config))
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
resource = create_wrapped_resource(webcls, config, app)
webservice = TCPServer(http_port,
server.Site(resource),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def setupWriterProcessor(root_service, settings):
from carbon import cache # Register CacheFeedingProcessor
from carbon.protocols import CacheManagementHandler
from carbon.writer import WriterService
from carbon import events
factory = ServerFactory()
factory.protocol = CacheManagementHandler
service = TCPServer(settings.CACHE_QUERY_PORT, factory, interface=settings.CACHE_QUERY_INTERFACE)
service.setServiceParent(root_service)
writer_service = WriterService()
writer_service.setServiceParent(root_service)
if settings.USE_FLOW_CONTROL:
events.cacheFull.addHandler(events.pauseReceivingMetrics)
events.cacheSpaceAvailable.addHandler(events.resumeReceivingMetrics)
def setup(self, cfg):
BasePlugin.setup(self, cfg)
self.factory = TelnetFactory(plugin=self)
self.factory.plugin = self
self.client = TCPServer(self.cfg.get("port", 9999), self.factory)
def getService(library, port=3689):
"""Return a DAAP server service instance attached to the given port."""
from twisted.application.internet import TCPServer
from twisted.web import server
Resource.library = library
return TCPServer(port, server.Site(server.Site(RootResource())))
def makeService(self, options):
"""
Create a service which will run a Gam3 server.
@param options: mapping of configuration
"""
from pygame.image import load
from gam3.network import Gam3Factory
from gam3.world import (
TCP_SERVICE_NAME, GAM3_SERVICE_NAME, Gam3Service, World)
from game.terrain import loadTerrainFromString, loadTerrainFromSurface
from twisted.python.filepath import FilePath
from twisted.internet import reactor
from twisted.application.service import MultiService
from twisted.protocols.policies import TrafficLoggingFactory
world = World(granularity=100, platformClock=reactor)
terrain = options['terrain']
if terrain:
if terrain.endswith('.png'):
voxels = loadTerrainFromSurface(load(terrain)).voxels
else:
raw = FilePath(terrain).getContent()
voxels = loadTerrainFromString(raw)
world.terrain.set(0, 0, 0, voxels)
service = MultiService()
factory = Gam3Factory(world)
if options['log-directory'] is not None:
factory = TrafficLoggingFactory(
factory, join(options['log-directory'], 'gam3'))
tcp = TCPServer(options['port'], factory)
tcp.setName(TCP_SERVICE_NAME)
tcp.setServiceParent(service)
gam3 = Gam3Service(world)
gam3.setName(GAM3_SERVICE_NAME)
gam3.setServiceParent(service)
return service
def makeService(config):
'''
Create service tree. Only one tracker can be used for every protocol!
@param config: instance of an Options class with configuration parameters.
@type config: C{twisted.python.usage.Options}
@return: service collection
@rtype: C{twisted.application.service.IServiceCollection}
'''
from gorynych.receiver.factories import ReceivingFactory
from gorynych.receiver import protocols, parsers
from gorynych.receiver.receiver import ReceiverRabbitQueue, ReceiverService, AuditFileLog
# Set up application.
application = service.Application("ReceiverServer")
sc = service.IServiceCollection(application)
# Prepare receiver.
audit_log = AuditFileLog('audit_log')
sender = ReceiverRabbitQueue(host='localhost', port=5672,
exchange='receiver', exchange_type='fanout')
parser = getattr(parsers, config['tracker'])()
sender.setName('RabbitMQReceiverService')
sender.setServiceParent(sc)
receiver_service = ReceiverService(sender, audit_log, parser)
receiver_service.setName('ReceiverService')
receiver_service.setServiceParent(sc)
if 'tcp' in config['protocols']:
receiver_server = TCPServer(config['port'],
ReceivingFactory(receiver_service))
protocol = getattr(protocols,
'_'.join((config['tracker'], 'tcp', 'protocol')))
receiver_server.args[1].protocol = protocol
receiver_server.setName(config['tracker'] + '_tcp')
receiver_server.setServiceParent(sc)
if 'udp' in config['protocols']:
protocol = getattr(protocols,
'_'.join((config['tracker'], 'udp', 'protocol')))(receiver_service)
receiver_server = UDPServer(config['port'], protocol)
receiver_server.setName(config['tracker'] + '_udp')
receiver_server.setServiceParent(sc)
return sc
def createCacheService(config):
from carbon.cache import MetricCache
from carbon.conf import settings
from carbon.protocols import CacheManagementHandler
# Configure application components
events.metricReceived.addHandler(MetricCache.store)
root_service = createBaseService(config)
factory = ServerFactory()
factory.protocol = CacheManagementHandler
service = TCPServer(int(settings.CACHE_QUERY_PORT), factory,
interface=settings.CACHE_QUERY_INTERFACE)
service.setServiceParent(root_service)
# have to import this *after* settings are defined
from carbon.writer import WriterService
service = WriterService()
service.setServiceParent(root_service)
if settings.USE_FLOW_CONTROL:
events.cacheFull.addHandler(events.pauseReceivingMetrics)
events.cacheSpaceAvailable.addHandler(events.resumeReceivingMetrics)
return root_service
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
schedpath = config.get('scheduler', 'scrapyd.scheduler.SpiderScheduler')
schedCls = load_object(schedpath)
scheduler = schedCls(config, app)
environment = Environment(config)
pubsub_path = config.get('pubsub', 'scrapyd.pubsub.BasePubSub')
pubsubCls = load_object(pubsub_path)
pubsub = pubsubCls(config, app)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
app.setComponent(IPubSub, pubsub)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint("http_port", 6800)
bind_address = config.get("bind_address", "0.0.0.0")
poll_interval = config.getfloat("poll_interval", 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get("launcher", "scrapyd.launcher.Launcher")
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get("webroot", "scrapyd.website.Root")
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port, server.Site(webcls(config, app)), interface=bind_address)
log.msg(
format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port,
)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application():
app = Application("Scrapyd")
config = Config()
http_port = config.getint('http_port', 6800)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def createCacheService(config):
from carbon.cache import MetricCache
from carbon.conf import settings
from carbon.protocols import CacheManagementHandler
# Configure application components
events.metricReceived.addHandler(MetricCache.store)
root_service = createBaseService(config)
factory = ServerFactory()
factory.protocol = CacheManagementHandler
service = TCPServer(int(settings.CACHE_QUERY_PORT), factory,
interface=settings.CACHE_QUERY_INTERFACE)
service.setServiceParent(root_service)
# have to import this *after* settings are defined
from carbon.writer import WriterService
service = WriterService()
service.setServiceParent(root_service)
# use_amqp_pub = settings.get("ENABLE_AMQP_PUB", False)
# if use_amqp_pub:
# from carbon import amqp_pub
# amqp_pub_host = settings.get("AMQP_PUB_HOST", "localhost")
# amqp_pub_port = settings.get("AMQP_PUB_PORT", 5672)
# amqp_pub_user = settings.get("AMQP_PUB_USER", "guest")
# amqp_pub_password = settings.get("AMQP_PUB_PASSWORD", "guest")
# amqp_pub_verbose = settings.get("AMQP_PUB_VERBOSE", False)
# amqp_pub_vhost = settings.get("AMQP_PUB_VHOST", "/")
# amqp_pub_spec = settings.get("AMQP_PUB_SPEC", None)
# amqp_pub_exchange_name = settings.get("AMQP_EXCHANGE", "graphite")
# factory = amqp_pub.createAMQPPublisher(
# amqp_pub_user, amqp_pub_password,
# vhost=amqp_pub_vhost, spec=amqp_pub_spec,
# exchange_name=amqp_pub_exchange_name,
# verbose=amqp_pub_verbose)
# service = TCPClient(amqp_pub_host, int(amqp_pub_port), factory)
# service.setServiceParent(root_service)
# from carbon.amqp_pub import ProducerService
# service = ProducerService()
# service.setServiceParent(root_service)
if settings.USE_FLOW_CONTROL:
events.cacheFull.addHandler(events.pauseReceivingMetrics)
events.cacheSpaceAvailable.addHandler(events.resumeReceivingMetrics)
return root_service
def createCacheService(config):
from carbon.cache import MetricCache
from carbon.conf import settings
from carbon.events import metricReceived
from carbon.protocols import protocolManager
from carbon.protocols import CacheQueryHandler
# Configure application components
metricReceived.installHandler(MetricCache.store)
root_service = createBaseService(config)
factory = protocolManager.createFactory(CacheQueryHandler)
service = TCPServer(int(settings.CACHE_QUERY_PORT), factory,
interface=settings.CACHE_QUERY_INTERFACE)
service.setServiceParent(root_service)
# have to import this *after* settings are defined
from carbon.writer import WriterService
service = WriterService()
service.setServiceParent(root_service)
return root_service
def startService(self):
if self.env.config.getbool('ssh', 'autostart'):
TCPServer.startService(self)
