def application(config, components=interfaces):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
for interface, key in interfaces:
path = config.get(key)
cls = load_object(path)
component = cls(config)
app.setComponent(interface, component)
poller = component
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
poll_every = config.getint("poll_every", 5)
timer = TimerService(poll_every, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = int(environ.get('PORT', config.getint('http_port', 6800)))
config.cp.set('scrapyd', 'database_url', environ.get('DATABASE_URL'))
poller = Psycopg2QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = Psycopg2SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
log.msg("Scrapyd web console available at http://localhost:%s/ (HEROKU)"
% http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = int(environ.get('PORT', config.getint('http_port', 6800)))
config.cp.set('scrapyd', 'database_url', environ.get('DATABASE_URL'))
poller = Psycopg2QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = Psycopg2SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
log.msg("Scrapyd web console available at http://localhost:%s/ (HEROKU)" %
http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
portal = Portal(PublicHTMLRealm(config, app), [FilePasswordDB(str(config.get('passwd', '')))])
credentialFactory = DigestCredentialFactory("md5", "Go away")
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(HTTPAuthSessionWrapper(portal, [credentialFactory])))
log.msg("Scrapyd web console available at http://localhost:%s/" % http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg("Scrapyd web console available at http://%s:%s/" % (bind_address, http_port))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
root = Root(config, app)
root = configRoot(root, config)
webservice = TCPServer(http_port, server.Site(root))
log.msg("Scrapyd web console available at http://localhost:%s/" % http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def get_application(config):
app = Application('Scrapyd')
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd_mongodb.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(
http_port, server.Site(Root(config, app)),
interface=bind_address)
log.msg('http://%(bind_address)s:%(http_port)s/' % {'bind_address':bind_address, 'http_port':http_port})
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.msg("Scrapyd web console available at http://%s:%s/" %
(bind_address, http_port))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def _get_application():
"""
Factory function that returns an Application object.
If the object does not exist then it creates a new Application object.
(Internal use only).
"""
global _application
if _application is not None:
return _application
_application = Application(consts.APP_NAME)
logfile = WaderLogFile(consts.LOG_NAME, consts.LOG_DIR,
maxRotatedFiles=consts.LOG_NUMBER)
_application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
return _application
def application(config):
app = Application("Flowder")
app_id = config.get('app_id', 'fw0')
logfile = config.get('logfile', '/var/log/flowder.log')
loglevel = config.get('loglevel', 'info')
db_file = config.get('db_file', 'flowder')
rest_port = config.getint('rest_port', 4000)
rest_bind = config.get('rest_bind', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 1)
poll_size = config.getint("poll_size", 5)
signalmanager = SignalManager()
app.setComponent(ISignalManager, signalmanager)
fetcher = FetcherService(config)
fetcher.setServiceParent(app)
poller = QueuePoller(app, poll_size)
poller.setServiceParent(app)
db_file = '%s.db' % db_file
task_storage = FileDownloaderTaskStorage(app, db_file)
task_storage.setServiceParent(app)
timer = TimerService(poll_interval, poller.poll)
timer.setServiceParent(app)
scheduler = TaskScheduler(config, app)
scheduler.setServiceParent(app)
laupath = config.get('launcher', 'flowder.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(app, config)
launcher.setServiceParent(app)
restService = TCPServer(rest_port,
server.Site(Root(app, config)),
interface=rest_bind)
restService.setServiceParent(app)
amqp_publisher = AmqpService(app, config)
amqp_publisher.setServiceParent(app)
log.msg("Starting Flowder services (;-)")
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
jspath = config.get('jobstorage', 'scrapyd.jobstorage.MemoryJobStorage')
jscls = load_object(jspath)
jobstorage = jscls(config)
app.setComponent(IJobStorage, jobstorage)
eggstorage = config.get('eggstorage',
'scrapyd.eggstorage.FilesystemEggStorage')
eggstoragecls = load_object(eggstorage)
app.setComponent(IEggStorage, eggstoragecls(config))
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
resource = create_wrapped_resource(webcls, config, app)
webservice = TCPServer(http_port,
server.Site(resource),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(webcls(config, app)),
interface=bind_address)
log.msg(
format=
"Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
host = get_host_ip(config)
redis_host = config.get('redis_host', 'localhost')
redis_port = config.get('redis_port', 6379)
redis_db = config.get('redis_db', 0)
redis_pool = redis.ConnectionPool(host=redis_host,
port=redis_port,
db=redis_db)
register_to_redis(config, redis_pool)
log.msg('Registering scrapyd [{}] to redis {}:{} at db {}'.format(
host, redis_host, redis_port, redis_db))
# log.msg('2018-11-03 10:10 am')
redis_interval = config.getfloat('redis_interval', 5)
register_timer = TimerService(redis_interval, register_to_redis, config,
redis_pool)
register_timer.setServiceParent(app)
return app
def application(settings):
app = Application("Scraperd")
poller = QueuePoller(settings)
environment = Environment(settings)
app.setComponent(IPoller, poller)
app.setComponent(IEnvironment, environment)
laupath = settings.get('DAEMON_LAUNCHER', 'scraper.daemon.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(settings, app)
poll_interval = settings.getfloat('DAEMON_POLL_INTERVAL', 5)
timer = TimerService(poll_interval, poller.poll)
launcher.setServiceParent(app)
timer.setServiceParent(app)
return app
def application(config):
app = Application("Spider Platform Slave")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
performance = Performance()
egg_storage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IPerformance, performance)
app.setComponent(IEggStorage, egg_storage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
lau_path = config.get('launcher', 'slave.launcher.Launcher')
lau_cls = load_object(lau_path)
launcher = lau_cls(config, app)
web_path = config.get('web_root', 'slave.website.Root')
web_cls = load_object(web_path)
timer_queue = TimerService(poll_interval, poller.poll)
timer_performance = TimerService(1, performance.poll)
webservice = TCPServer(http_port, server.Site(web_cls(config, app)),
interface=bind_address)
log.msg(
format="Spider Platform Slave web console available at "
"http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer_queue.setServiceParent(app)
timer_performance.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webpath = config.get('webroot', 'scrapyd.website.Root')
webcls = load_object(webpath)
username = config.get('username', '')
password = config.get('password', '')
if username and password:
if ':' in username:
sys.exit("The `username` option contains illegal character ':', "
"check and update the configuration file of Scrapyd")
portal = Portal(PublicHTMLRealm(webcls(config, app)),
[StringCredentialsChecker(username, password)])
credential_factory = BasicCredentialFactory("Auth")
resource = HTTPAuthSessionWrapper(portal, [credential_factory])
log.msg("Basic authentication enabled")
else:
resource = webcls(config, app)
log.msg("Basic authentication disabled as either `username` or `password` is unset")
webservice = TCPServer(http_port, server.Site(resource), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
schedpath = config.get('scheduler', 'scrapyd.scheduler.SpiderScheduler')
schedCls = load_object(schedpath)
scheduler = schedCls(config, app)
environment = Environment(config)
pubsub_path = config.get('pubsub', 'scrapyd.pubsub.BasePubSub')
pubsubCls = load_object(pubsub_path)
pubsub = pubsubCls(config, app)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
app.setComponent(IPubSub, pubsub)
laupath = config.get('launcher', 'scrapyd.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config, app)
taskpoller = TaskPoller(config, app) #
app.setComponent(IPoller, poller)
app.setComponent(IPoller, taskpoller) #
timer = TimerService(poll_interval, poller.poll)
tasktimer = TimerService(poll_interval, taskpoller.poll) #
timer.setServiceParent(app)
tasktimer.setServiceParent(app) #
http_port = config.getint('http_port', 9090)
bind_address = config.get('bind_address', '127.0.0.1')
log.msg(format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address, http_port=http_port)
webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address)
webservice.setServiceParent(app)
return app
def application(config):
"""
提供http服务并启动应用
:param config:
:return:
"""
app = Application("engine")
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '127.0.0.1')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
# metrics = MetricsReporter(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
# app.setComponent(IMetrics, metrics)
laupath = config.get('launcher', 'engine.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get('webroot', 'engine.website.Root')
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(webcls(config, app)),
interface=bind_address)
log.msg(
format="DtCrawlEngine 访问地址 at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application():
app = Application("Scrapyd")
config = Config()
http_port = config.getint('http_port', 6800)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
launcher = Launcher(config, app)
timer = TimerService(5, poller.poll)
webservice = TCPServer(http_port, server.Site(Root(config, app)))
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def application(config):
app = Application("Scrapyd")
http_port = config.getint("http_port", 6800)
bind_address = config.get("bind_address", "0.0.0.0")
poll_interval = config.getfloat("poll_interval", 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get("launcher", "scrapyd.launcher.Launcher")
laucls = load_object(laupath)
launcher = laucls(config, app)
webpath = config.get("webroot", "scrapyd.website.Root")
webcls = load_object(webpath)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port, server.Site(webcls(config, app)), interface=bind_address)
log.msg(
format="Scrapyd web console available at http://%(bind_address)s:%(http_port)s/",
bind_address=bind_address,
http_port=http_port,
)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
mortal = Bot("Mortal", Room.get(name__icontains = "mortals"))
GarenaRSFactory(mortal, Account.get(login = "******"),
write_only = False)
GarenaRSFactory(mortal, Account.get(login = "******"), send_kicks = True)
GarenaRSFactory(mortal, Account.get(login = "******"))
GarenaRSFactory(mortal, Account.get(login = "******"))
#god = Bot("God", Room.get(pk = 196880))
#GarenaRSFactory(god, Account.get(login = "******"),
# write_only = False, send_kicks = True)
from twisted.application.service import Application
application = Application("OSPL|Bot")
from twisted.python.log import ILogObserver, FileLogObserver
from twisted.python.logfile import LogFile
logfile = LogFile("twisted.log", "tmp/", maxRotatedFiles = 5)
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
if __name__ == '__main__':
reactor.run()
from twisted.application.service import Application
from twisted.python.log import ILogObserver, FileLogObserver
from twisted.python.logfile import DailyLogFile
application = Application("Pollapli")
logfile = DailyLogFile("pollapli.log", ".")#"/tmp")
application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
def create_application(args=None, twistd_user=None, log_group=None):
home_dir = util.get_home_dir()
os.chdir(home_dir)
is_logging_insecure = False
# parse command-line args, if appropriate
primary_only_time = None
if args:
option_parser = argparse.ArgumentParser()
option_parser.add_argument(
"--primary-only",
type=int,
nargs="?",
help=
"This option disables secondary authentication for the specified number of minutes (default 60)",
default=None,
const=60,
)
option_parser.add_argument(
"--logging-insecure",
action="store_true",
help=
"This option enables debug, and prints logs containing passwords and possibly other secrets.",
default=False,
)
options = option_parser.parse_args()
is_logging_insecure = options.logging_insecure
primary_only_time = options.primary_only
config_filename = os.path.join("conf", "authproxy.cfg")
configuration = config_provider.get_config(config_filename)
if primary_only_time is not None:
if primary_only_time > 240:
print(
"Primary only mode can only be enabled for a maximum of 4 hours (240 minutes)"
)
sys.exit(2)
else:
PrimaryOnlyManager.enable_primary_only(primary_only_time)
main_config = configuration.get_main_section_config()
if main_config:
log.msg("Main Configuration:")
log.config(main_config)
fips_mode = main_config.get_bool("fips_mode", False)
if fips_mode:
fips_manager.enable()
# Set up our observers
if is_logging_insecure:
observers = [textFileLogObserver(sys.stdout)]
else:
observers = log_observation.get_observers(main_config, twistd_user,
log_group)
for observer in observers:
globalLogPublisher.addObserver(observer)
# Global debug mode
if is_logging_insecure:
debug_mode = True
else:
debug_mode = main_config.get_bool("debug", False)
http.set_debug(debug_mode)
http.set_is_logging_insecure(is_logging_insecure)
# Create main application.
application = Application("duoauthproxy")
LogReadyService().setServiceParent(application)
fips_mode = fips_manager.status()
if fips_mode:
log.msg("FIPS mode {0} is enabled with {1}".format(
fips_mode, fips_manager.get_openssl_version()))
else:
log.msg("FIPS mode is not enabled")
# get ca certs file
http_ca_certs_file = main_config.get_str("http_ca_certs_file", "")
if http_ca_certs_file:
http_ca_certs_file = util.resolve_file_path(http_ca_certs_file)
else:
http_ca_certs_file = os.path.join("conf",
const.DEFAULT_HTTP_CERTS_FILE)
# read ca certs
if not os.path.isfile(http_ca_certs_file):
http_ca_certs_file = os.path.join("conf", http_ca_certs_file)
with open(http_ca_certs_file, "r") as bundle_fp:
http.set_ca_certs(ssl_verify.load_ca_bundle(bundle_fp))
# get proxy settings
http_proxy_host = main_config.get_str("http_proxy_host", "")
http_proxy_port = main_config.get_int("http_proxy_port", 80)
if http_proxy_host:
http.set_proxy(http_proxy_host, http_proxy_port)
sections = section.parse_sections(configuration, is_logging_insecure)
module_factory = section.ModuleFactory(sections, application)
modules_by_type = module_factory.make_modules()
if not any(modules_by_type.values()):
raise config_error.ConfigError("No integrations in config file.")
# Setup forwarding/server pairs by port
for port, interface in modules_by_type.get("server", []):
server_networks = {}
server_names = {}
for section_name, server_module, server_config in modules_by_type[
"server"][(port, interface)]:
client_name = configuration.get_section_client(section_name)
if not client_name:
if server_module.Module.no_client:
modules_by_type["client"]["no_client"] = None
client_name = "no_client"
else:
raise config_error.ConfigError(
'Neither module %s or main has "client" value' %
section_name)
if section_name.startswith(
"ldap_server_auto"
) and not client_name.startswith("ad_client"):
raise config_error.ConfigError(
"ad_client is required by ldap_server_auto. No ad_client found in config file. "
)
if client_name != "radius_client" and server_config.get_str(
"pass_through_attr_names", ""):
raise config_error.ConfigError(
"Can only pass through radius attributes if using a radius client"
)
server_instance = server_module.Module(
server_config, modules_by_type["client"][client_name],
section_name)
server_instance.setServiceParent(application)
if section_name.startswith("radius_server_"):
server_networks[server_instance] = parse_radius_secrets(
server_config).keys()
server_names[server_instance] = section_name
if server_names:
forward_module = forward_serv
forward_instance = forward_module.Module(
port=port,
servers=server_networks,
server_names=server_names,
interface=interface,
debug=debug_mode,
)
forward_instance.setServiceParent(application)
# set user-agent
sections = ",".join(sorted(set(configuration.list_sections())))
user_agent = "duoauthproxy/{0} ({1}; Python{2}; {3})".format(
get_version(), platform.platform(), platform.python_version(),
sections)
http.set_user_agent(user_agent)
# Authproxy uses globalLogPublisher to emit events. Defining a no-op emitter will squelch the creation
# of the unwatned twistd default logging mechanisms.
def no_op_emitter(eventDict):
pass
application.setComponent(ILogObserver, no_op_emitter)
return application
def create_application(args=None, twistd_user=None, log_group=None):
home_dir = util.get_home_dir()
os.chdir(home_dir)
is_logging_insecure = False
if syslog is not None:
facility_dict = {
'LOG_KERN': pySyslog.LOG_KERN,
'LOG_USER': pySyslog.LOG_USER,
'LOG_MAIL': pySyslog.LOG_MAIL,
'LOG_DAEMON': pySyslog.LOG_DAEMON,
'LOG_AUTH': pySyslog.LOG_AUTH,
'LOG_LPR': pySyslog.LOG_LPR,
'LOG_NEWS': pySyslog.LOG_NEWS,
'LOG_UUCP': pySyslog.LOG_UUCP,
'LOG_CRON': pySyslog.LOG_CRON,
'LOG_SYSLOG': pySyslog.LOG_SYSLOG,
'LOG_LOCAL0': pySyslog.LOG_LOCAL0,
'LOG_LOCAL1': pySyslog.LOG_LOCAL1,
'LOG_LOCAL2': pySyslog.LOG_LOCAL2,
'LOG_LOCAL3': pySyslog.LOG_LOCAL3,
'LOG_LOCAL4': pySyslog.LOG_LOCAL4,
'LOG_LOCAL5': pySyslog.LOG_LOCAL5,
'LOG_LOCAL6': pySyslog.LOG_LOCAL6,
'LOG_LOCAL7': pySyslog.LOG_LOCAL7
}
# parse command-line args, if appropriate
primary_only_time = None
if args:
option_parser = argparse.ArgumentParser()
option_parser.add_argument(
"--primary-only", type=int, nargs='?',
help="This option disables secondary authentication for the specified number of minutes (default 60)",
default=None, const=60
)
option_parser.add_argument(
"--logging-insecure",
action="store_true",
help="This option enables debug, and prints logs containing passwords and possibly other secrets.",
default=False
)
options = option_parser.parse_args()
is_logging_insecure = options.logging_insecure
primary_only_time = options.primary_only
config_filename = os.path.join('conf', 'authproxy.cfg')
configuration = config_provider.get_config(config_filename)
if primary_only_time is not None:
if primary_only_time > 240:
print("Primary only mode can only be enabled for a maximum of 4 hours (240 minutes)")
sys.exit(2)
else:
PrimaryOnlyManager.enable_primary_only(primary_only_time)
main_config = configuration.get_main_section_config()
if main_config:
log.msg('Main Configuration:')
log.config(main_config)
fips_mode = main_config.get_bool('fips_mode', False)
if fips_mode:
fips_manager.enable()
# handle log configuration
log_to_file = main_config.get_bool('log_file', False)
log_stdout = main_config.get_bool('log_stdout', False)
log_syslog = main_config.get_bool('log_syslog', False)
log_auth_events = main_config.get_bool('log_auth_events', False)
log_sso_events = main_config.get_bool('log_sso_events', True)
if is_logging_insecure:
globalLogPublisher.addObserver(textFileLogObserver(sys.stdout))
else:
if log_to_file or not (log_to_file or log_syslog or log_stdout):
log_dir = main_config.get_str('log_dir', 'log')
log_max_size = main_config.get_int('log_max_size', 10 * (1 << 20))
log_max_files = main_config.get_int('log_max_files', 6)
if log_max_files == 0:
# we need to pass None explicitly if we want there to be no limit
# 0 would just mean no logfiles would get kept...
log_max_files = None
log_file = create_log_file('authproxy.log', log_dir, log_max_size,
log_max_files, twistd_user, log_group)
log_file_observer = textFileLogObserver(log_file)
if log_auth_events:
auth_log_file = create_log_file('authevents.log', log_dir, log_max_size, log_max_files, twistd_user, log_group)
auth_observer = FileLogObserver(auth_log_file, log.format_auth_event)
else:
auth_observer = log.no_op_observer
if log_sso_events:
sso_log_file = create_log_file('ssoevents.log', log_dir, log_max_size, log_max_files, twistd_user, log_group)
sso_observer = FileLogObserver(sso_log_file, log.format_sso_event)
else:
sso_observer = log.no_op_observer
auth_filtering_observer = FilteringLogObserver(auth_observer,
[log.auth_type_predicate],
log.no_op_observer)
globalLogPublisher.addObserver(auth_filtering_observer)
sso_filtering_observer = FilteringLogObserver(sso_observer,
[log.sso_type_predicate],
log.no_op_observer)
globalLogPublisher.addObserver(sso_filtering_observer)
# the default authproxy.log
log_file_observer = FilteringLogObserver(log_file_observer,
[log.only_default_log_predicate],
log.no_op_observer)
globalLogPublisher.addObserver(log_file_observer)
if log_stdout:
std_out_observer = textFileLogObserver(sys.stdout)
std_out_filter = FilteringLogObserver(log.no_op_observer,
[log.auth_type_predicate],
std_out_observer)
globalLogPublisher.addObserver(std_out_filter)
if log_syslog:
if syslog is None:
raise config_error.ConfigError('syslog not supported on Windows')
syslog_facilitystr = main_config.get_str('syslog_facility', 'LOG_USER')
syslog_facility = facility_dict.get(syslog_facilitystr, None)
if syslog_facility is None:
raise config_error.ConfigError('Unknown syslog_facility: {0}'.format(syslog_facilitystr))
syslog_observer = syslog.SyslogObserver('Authproxy', facility=syslog_facility)
wrapped_syslog_observer = LegacyLogObserverWrapper(syslog_observer.emit)
syslog_filtering_observer = FilteringLogObserver(log.no_op_observer,
[log.auth_type_predicate],
wrapped_syslog_observer)
globalLogPublisher.addObserver(syslog_filtering_observer)
# Global debug mode
if is_logging_insecure:
debug_mode = True
else:
debug_mode = main_config.get_bool('debug', False)
http.set_debug(debug_mode)
http.set_is_logging_insecure(is_logging_insecure)
# Create main application.
application = Application('duoauthproxy')
LogReadyService().setServiceParent(application)
fips_mode = fips_manager.status()
if fips_mode:
log.msg("FIPS mode {0} is enabled with {1}".format(fips_mode, fips_manager.get_openssl_version()))
else:
log.msg("FIPS mode is not enabled")
# get ca certs file
http_ca_certs_file = main_config.get_str('http_ca_certs_file', '')
if http_ca_certs_file:
http_ca_certs_file = util.resolve_file_path(http_ca_certs_file)
else:
http_ca_certs_file = os.path.join('conf', const.DEFAULT_HTTP_CERTS_FILE)
# read ca certs
if not os.path.isfile(http_ca_certs_file):
http_ca_certs_file = os.path.join('conf', http_ca_certs_file)
with open(http_ca_certs_file, 'r') as bundle_fp:
http.set_ca_certs(ssl_verify.load_ca_bundle(bundle_fp))
# get proxy settings
http_proxy_host = main_config.get_str('http_proxy_host', '')
http_proxy_port = main_config.get_int('http_proxy_port', 80)
if http_proxy_host:
http.set_proxy(http_proxy_host, http_proxy_port)
sections = section.parse_sections(configuration, is_logging_insecure)
module_factory = section.ModuleFactory(sections, application)
modules_by_type = module_factory.make_modules()
if not any(modules_by_type.values()):
raise config_error.ConfigError('No integrations in config file.')
# Setup forwarding/server pairs by port
for port, interface in modules_by_type.get('server', []):
server_networks = {}
server_names = {}
for section_name, server_module, server_config in modules_by_type['server'][(port, interface)]:
client_name = configuration.get_section_client(section_name)
if not client_name:
if server_module.Module.no_client:
modules_by_type['client']['no_client'] = None
client_name = 'no_client'
else:
raise config_error.ConfigError('Neither module %s or main has "client" value' % section_name)
if section_name.startswith('ldap_server_auto') and not client_name.startswith('ad_client'):
raise config_error.ConfigError('ad_client is required by ldap_server_auto. No ad_client found in config file. ')
if client_name != 'radius_client' \
and server_config.get_str('pass_through_attr_names', ''):
raise config_error.ConfigError('Can only pass through radius attributes if using a radius client')
server_instance = server_module.Module(server_config,
modules_by_type['client'][client_name],
section_name)
server_instance.setServiceParent(application)
if section_name.startswith('radius_server_'):
server_networks[server_instance] = parse_radius_secrets(server_config).keys()
server_names[server_instance] = section_name
if server_names:
forward_module = forward_serv
forward_instance = forward_module.Module(
port=port,
servers=server_networks,
server_names=server_names,
interface=interface,
debug=debug_mode,
)
forward_instance.setServiceParent(application)
# set user-agent
sections = ','.join(sorted(set(configuration.list_sections())))
user_agent = "duoauthproxy/{0} ({1}; Python{2}; {3})".format(
get_version(),
platform.platform(),
platform.python_version(),
sections)
http.set_user_agent(user_agent)
# Authproxy uses globalLogPublisher to emit events. Defining a no-op emitter will squelch the creation
# of the unwatned twistd default logging mechanisms.
def no_op_emitter(eventDict):
pass
application.setComponent(ILogObserver, no_op_emitter)
return application
