def launch_stdin_uploader():
from twisted.internet import reactor
logging.basicConfig(level=logging.WARNING, filename="ul.log")
if len(sys.argv) == 4:
uploader = LBRYStdinUploader(int(sys.argv[2]), int(sys.argv[3]), [])
elif len(sys.argv) == 6:
uploader = LBRYStdinUploader(int(sys.argv[2]), int(sys.argv[3]), [(sys.argv[4], int(sys.argv[5]))])
else:
print "Usage: lbrynet-stdin-uploader <stream_name> <peer_port> <dht_node_port>" \
" [<dht_bootstrap_host> <dht_bootstrap port>]"
sys.exit(1)
def start_stdin_uploader():
return uploader.start_live_stream(sys.argv[1])
def shut_down():
logging.debug("Telling the reactor to stop in 60 seconds")
reactor.callLater(60, reactor.stop)
d = task.deferLater(reactor, 0, uploader.start)
d.addCallback(lambda _: start_stdin_uploader())
d.addCallback(lambda _: shut_down())
reactor.addSystemEventTrigger('before', 'shutdown', uploader.shut_down)
reactor.run()
def onConnect():
# if keyAgent and options['agent']:
# cc = protocol.ClientCreator(reactor, SSHAgentForwardingLocal, conn)
# cc.connectUNIX(os.environ['SSH_AUTH_SOCK'])
if hasattr(conn.transport, 'sendIgnore'):
_KeepAlive(conn)
if options.localForwards:
for localPort, hostport in options.localForwards:
s = reactor.listenTCP(localPort,
forwarding.SSHListenForwardingFactory(conn,
hostport,
SSHListenClientForwardingChannel))
conn.localForwards.append(s)
if options.remoteForwards:
for remotePort, hostport in options.remoteForwards:
log.msg('asking for remote forwarding for %s:%s' %
(remotePort, hostport))
conn.requestRemoteForwarding(remotePort, hostport)
reactor.addSystemEventTrigger('before', 'shutdown', beforeShutdown)
if not options['noshell'] or options['agent']:
conn.openChannel(SSHSession())
if options['fork']:
if os.fork():
os._exit(0)
os.setsid()
for i in range(3):
try:
os.close(i)
except OSError as e:
import errno
if e.errno != errno.EBADF:
raise
def tearDown(self):
self._db1.close()
self._db2.close()
self._soledad.close()
# restore paths
os.environ["PATH"] = self.old_path
os.environ["HOME"] = self.old_home
def _delete_temporary_dirs():
# XXX should not access "private" attrs
for f in [self._soledad.local_db_path,
self._soledad.secrets.secrets_path]:
if os.path.isfile(f):
os.unlink(f)
# The following snippet comes from BaseLeapTest.setUpClass, but we
# repeat it here because twisted.trial does not work with
# setUpClass/tearDownClass.
soledad_assert(
self.tempdir.startswith('/tmp/leap_tests-'),
"beware! tried to remove a dir which does not "
"live in temporal folder!")
shutil.rmtree(self.tempdir)
from twisted.internet import reactor
reactor.addSystemEventTrigger(
"after", "shutdown", _delete_temporary_dirs)
def __init__(self, options):
"""
@param options: the optparsed dictionary of command-line options
@type options: an object with attributes
"""
self.options = options
self.workerName = options.name
# the last port is reserved for our FeedServer
if not self.options.randomFeederports:
self.ports = self.options.feederports[:-1]
else:
self.ports = []
self.medium = medium.WorkerMedium(self)
# really should be componentJobHeaven, but this is shorter :)
self.jobHeaven = job.ComponentJobHeaven(self)
# for ephemeral checks
self.checkHeaven = job.CheckJobHeaven(self)
self.managerConnectionInfo = None
# it's possible we don't have a feed server, if we are
# configured to have 0 tcp ports; setup this in listen()
self.feedServer = None
self.stopping = False
reactor.addSystemEventTrigger('before', 'shutdown',
self.shutdownHandler)
self._installHUPHandler()
def applicationShouldTerminate_(self, sender):
if reactor.running:
reactor.addSystemEventTrigger(
'after', 'shutdown', AppHelper.stopEventLoop)
reactor.stop()
return False
return True
def __init__(self, application):
"""
Setup the server.
application - an instantiated Twisted application
"""
sys.path.insert(1, '.')
# create a store of services
self.services = service.IServiceCollection(application)
self.amp_protocol = None # set by amp factory
self.sessions = SESSIONS
self.sessions.server = self
# Database-specific startup optimizations.
self.sqlite3_prep()
self.start_time = time.time()
# Run the initial setup if needed
self.run_initial_setup()
# initialize channelhandler
channelhandler.CHANNELHANDLER.update()
# set a callback if the server is killed abruptly,
# by Ctrl-C, reboot etc.
reactor.addSystemEventTrigger('before', 'shutdown',
self.shutdown, _reactor_stopping=True)
self.game_running = True
# track the server time
self.run_init_hooks()
def wsgi_resource():
pool = threadpool.ThreadPool()
pool.start()
# Allow Ctrl-C to get you out cleanly:
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
wsgi_resource = wsgi.WSGIResource(reactor, pool, get_internal_wsgi_application())
return wsgi_resource
def get_WHAT_resource(deployment_type, port=None):
'''
Pseudo factory that returns the proper Resource object for the WHAT.
Takes a deployment type and (for development) a port number.
Returns a tuple (Twisted Resource, Twisted Application, Twisted Server)
(This function can almost get away with returning just the resource, but we sometimes need to daemonize the server).
'''
print "Deployment type: %s" % deployment_type
if not port: #For development, port will have been specified.
if deployment_type == "production":
port = PRODUCTION_SERVER_PORT
#TODO: Integrate SSL.
elif deployment_type == "staging":
port = STAGING_SERVER_PORT
else: #If somehow the deployment type wasn't specified properly, let's crash now rather than wait for a problem in django.
wsgiThreadPool.stop()
exit('Deployment type must be "production," "staging," or "development." \nIf you need help, talk to the Dev / NetOps satchem.')
application = service.Application('SlashRoot WHAT in %s' % deployment_type)
# Create and start a thread pool,
wsgiThreadPool = ThreadPool()
#The pool will stop when the reactor shuts down
reactor.addSystemEventTrigger('after', 'shutdown', wsgiThreadPool.stop)
what_server = service.MultiService()
tps = ThreadPoolService(wsgiThreadPool)
tps.setServiceParent(what_server)
#Use django's WSGIHandler to create the resource.
what_django_resource = WSGIResource(reactor, tps.pool, WSGIHandler())
root = Root(what_django_resource)
#Now we need to handle static media.
#Servce Django media files off of /media:
if deployment_type == "development":
#TODO: Grab static media locations for WHAT files and django admin files from local settings and use them here.
#raise RuntimeError('Dominick, fix this.')
#Maybe we want to hardcode production and staging paths. Maybe we don't.
admin_static = MediaService(os.path.join(os.path.abspath("."), resources.DEVELOPMENT_ADMIN_MEDIA))
staticrsrc = MediaService(os.path.join(os.path.abspath("."), "%s/static" % PROJECT_ROOT))
else:
#Maybe we want to hardcode production and staging paths. Maybe we don't.
admin_static = MediaService(os.path.join(os.path.abspath("."), resources.PRODUCTION_ADMIN_MEDIA))
staticrsrc = MediaService(os.path.join(os.path.abspath("."), resources.STATIC_PRODUCTION))
#Now that we have the static media locations, add them to the root.
root.putChild("media_admin", admin_static)
root.putChild("media", staticrsrc)
main_site = server.Site(root)
internet.TCPServer(port, main_site).setServiceParent(what_server)
#what_server = strports.service('tcp:%s' % port, server.Site(resource)) #TODO: Use port number from kwarg
return what_django_resource, application, what_server
def launch_stdout_downloader():
from twisted.internet import reactor
logging.basicConfig(level=logging.WARNING, filename="dl.log")
if len(sys.argv) == 3:
downloader = StdoutDownloader(int(sys.argv[2]), [])
elif len(sys.argv) == 5:
downloader = StdoutDownloader(int(sys.argv[2]), [(sys.argv[3], int(sys.argv[4]))])
else:
print "Usage: lbrynet-stdout-downloader <sd_hash> <peer_port> <dht_node_port>" \
" [<dht_bootstrap_host> <dht_bootstrap port>]"
sys.exit(1)
def start_stdout_downloader():
return downloader.start_download(sys.argv[1])
def print_error(err):
logging.warning(err.getErrorMessage())
def shut_down():
reactor.stop()
d = task.deferLater(reactor, 0, downloader.start)
d.addCallback(lambda _: start_stdout_downloader())
d.addErrback(print_error)
d.addCallback(lambda _: shut_down())
reactor.addSystemEventTrigger('before', 'shutdown', downloader.shut_down)
reactor.run()
def setup(self, opts, restrict=None, shelveoperators=False, cache=True, raw=False):
self.load_chunk_size = datetime.timedelta(hours=int(opts.get('ChunkSize', 24)))
self.source_url = opts.get('SourceUrl', 'http://new.openbms.org/backend')
# self.source_url = opts.get('SourceUrl', 'http://ar1.openbms.org:8079')
if not raw and restrict:
self.restrict = '(' + restrict + ') and not has Metadata/Extra/Operator'
else:
self.restrict = restrict
if shelveoperators:
self.operators = shelve.open(opts.get('OperatorCache', '.operators'),
protocol=2, writeback=True)
# sync the operator state periodically and at exit
util.periodicCallInThread(self.operators.sync).start(60)
reactor.addSystemEventTrigger('after', 'shutdown',
self.operators.close)
else:
self.operators = {}
self.oplist = []
self.arclient = SmapClient(self.source_url)
self.cache = cache
# create timeseries from cached operator state
for sid, oplist in self.operators.iteritems():
for path, op in oplist.itervalues():
self.add_operator(path, op)
def GetworkProxy_main(cb):
log.info("Stratum proxy version %s Connecting to Pool..." % version.VERSION)
# Connect to Stratum pool
f = SocketTransportClientFactory(settings.HOSTNAME, settings.LISTEN_SOCKET_TRANSPORT,
debug=False, proxy=None, event_handler=client_service.ClientMiningService)
job_registry = jobs.JobRegistry(f, cmd='', no_midstate=settings.GW_DISABLE_MIDSTATE, real_target=settings.GW_SEND_REAL_TARGET)
client_service.ClientMiningService.job_registry = job_registry
client_service.ClientMiningService.reset_timeout()
workers = worker_registry.WorkerRegistry(f)
f.on_connect.addCallback(on_connect, workers, job_registry)
f.on_disconnect.addCallback(on_disconnect, workers, job_registry)
# Cleanup properly on shutdown
reactor.addSystemEventTrigger('before', 'shutdown', on_shutdown, f)
# Block until proxy connects to the pool
yield f.on_connect
# Setup getwork listener
gw_site = Site(getwork_listener.Root(job_registry, workers,
stratum_host=settings.HOSTNAME, stratum_port=settings.LISTEN_SOCKET_TRANSPORT,
custom_lp=False, custom_stratum=False,
custom_user=False, custom_password=False
))
gw_site.noisy = False
reactor.listenTCP(settings.GW_PORT, gw_site, interface='0.0.0.0')
log.info("Getwork Proxy is online, Port: %d" % (settings.GW_PORT))
def start_app_start(self):
"""Start the app for the start subcommand."""
config = self.master_config
# First see if the cluster is already running
try:
pid = self.get_pid_from_file()
except PIDFileError:
pass
else:
self.log.critical(
'Cluster is already running with [pid=%s]. '
'use "ipcluster stop" to stop the cluster.' % pid
)
# Here I exit with a unusual exit status that other processes
# can watch for to learn how I existed.
self.exit(ALREADY_STARTED)
# Now log and daemonize
self.log.info(
'Starting ipcluster with [daemon=%r]' % config.Global.daemonize
)
# TODO: Get daemonize working on Windows or as a Windows Server.
if config.Global.daemonize:
if os.name=='posix':
daemonize()
# Now write the new pid file AFTER our new forked pid is active.
self.write_pid_file()
reactor.addSystemEventTrigger('during','shutdown', self.remove_pid_file)
reactor.run()
def __init__(self, endpoint, maxClients=5, reactor=None):
"""
@param endpoint: An L{IStreamClientEndpoint} indicating the server to
connect to.
@param maxClients: A C{int} indicating the maximum number of clients.
@param reactor: An L{IReactorTCP} provider used to initiate new
connections.
"""
self._endpoint = endpoint
self._maxClients = maxClients
if reactor is None:
from twisted.internet import reactor
self._reactor = reactor
self.shutdown_deferred = None
self.shutdown_requested = False
reactor.addSystemEventTrigger(
'before', 'shutdown', self._shutdownCallback
)
self._busyClients = set([])
self._freeClients = set([])
self._pendingConnects = 0
self._commands = []
def main(args=None):
"""
Starts the daemon.
:param list args: List of the console parameters
"""
if args is None:
args = sys.argv[1:]
pid = os.getpid()
logging.basicConfig(filename=os.path.join(conf.LOGGING_DIR, 'polod.log'),
level=conf.LOGGING_LEVEL.upper(),
format=conf.LOGGING_FORMAT)
#try:
# f = open(conf.PIDFILE_POLO, 'w')
# f.write(str(pid))
# f.close()
#except Exception as e:
# logging.error(e)
# sys.exit(1)
signal.signal(signal.SIGHUP, signal.SIG_IGN)
signal.signal(signal.SIGUSR1, reload_services)
reactor.addSystemEventTrigger('before', 'shutdown', graceful_shutdown)
reactor.callWhenRunning(start_multicast)
reactor.callWhenRunning(start_binding)
reactor.run()
def connect_tunnel(tunnel_id,
base_url,
username,
access_key,
local_host,
remote_host,
ports,
connected_callback,
shutdown_callback,
diagnostic):
def check_n_call():
global open_tunnels
open_tunnels += 1
if open_tunnels >= len(ports) and connected_callback:
connected_callback()
for (local_port, remote_port) in ports:
d = protocol.ClientCreator(reactor,
TunnelTransport,
username,
access_key,
local_host,
local_port,
remote_port,
check_n_call,
diagnostic).connectTCP(remote_host, 22)
reactor.addSystemEventTrigger("before", "shutdown", shutdown_callback)
def start_reactor(self):
reactor.callWhenRunning(
lambda: self.log.info('twisted-reactor-started'))
reactor.addSystemEventTrigger('before', 'shutdown',
self.shutdown_components)
reactor.run()
def __init__(self, application):
"""
Setup the server.
Args:
application (Application): An instantiated Twisted application
"""
sys.path.append('.')
# create a store of services
self.services = service.IServiceCollection(application)
self.amp_protocol = None # set by amp factory
self.sessions = PORTAL_SESSIONS
self.sessions.portal = self
self.process_id = os.getpid()
self.server_process_id = None
self.server_restart_mode = "shutdown"
self.server_info_dict = {}
# in non-interactive portal mode, this gets overwritten by
# cmdline sent by the evennia launcher
self.server_twistd_cmd = self._get_backup_server_twistd_cmd()
# set a callback if the server is killed abruptly,
# by Ctrl-C, reboot etc.
reactor.addSystemEventTrigger('before', 'shutdown',
self.shutdown, _reactor_stopping=True, _stop_server=True)
def __init__(self):
# Check for any other instances
if config.pid and os.name != "posix":
config.pid = ""
if config.pid:
twistd.checkPID(config.pid)
# Do any auto-update stuff
xdb.housekeep()
# Daemonise the process and write the PID file
if daemonizeme and os.name == "posix":
twistd.daemonize()
if config.pid:
self.writePID()
jid = config.jid
if config.useXCP and config.compjid: jid = config.compjid
if config.saslUsername:
import sasl
self.c = sasl.buildServiceManager(jid, config.saslUsername, config.secret, "tcp:%s:%s" % (config.mainServer, config.port))
else:
self.c = component.buildServiceManager(jid, config.secret, "tcp:%s:%s" % (config.mainServer, config.port))
self.transportSvc = PyTransport()
self.transportSvc.setServiceParent(self.c)
self.c.startService()
reactor.addSystemEventTrigger('before', 'shutdown', self.shuttingDown)
def __init__(self, db_conn, hs):
self.client_ip_last_seen = Cache(
name="client_ip_last_seen",
keylen=4,
max_entries=50000 * CACHE_SIZE_FACTOR,
)
super(ClientIpStore, self).__init__(db_conn, hs)
self.register_background_index_update(
"user_ips_device_index",
index_name="user_ips_device_id",
table="user_ips",
columns=["user_id", "device_id", "last_seen"],
)
self.register_background_index_update(
"user_ips_last_seen_index",
index_name="user_ips_last_seen",
table="user_ips",
columns=["user_id", "last_seen"],
)
# (user_id, access_token, ip) -> (user_agent, device_id, last_seen)
self._batch_row_update = {}
self._client_ip_looper = self._clock.looping_call(
self._update_client_ips_batch, 5 * 1000
)
reactor.addSystemEventTrigger("before", "shutdown", self._update_client_ips_batch)
def __init__(self, config):
self.config = config
self.step = self.config[self.name]['step']
self.call = (self.update_feed_list, [], {})
self.pool = threadpool.ThreadPool(name=self.name)
reactor.callWhenRunning(self.pool.start)
reactor.addSystemEventTrigger('after', 'shutdown', self.pool.stop)
def __init__(self,service_config, hwios_config):
wsgi_handler = WSGIHandler()
self.hwios_config = hwios_config
self.config = service_config
if sys.argv[0] == 'tools/autoreload-twistd.py':
root = RootResource(self.single_threaded_wsgi_resource(wsgi_handler))
else:
root = RootResource(self.multi_threaded_wsgi_resource(wsgi_handler))
root.putChild("dav", WebDAV(self).get_resource())
root.putChild("media", StaticFile(os.path.join(os.path.join(self.config.location, 'media'))))
root.putChild("docs", StaticFile(os.path.join(os.path.join(self.config.location, '../','../','docs','_build','html'))))
site = WebSocketSite(root)
HWIOS.ws_realm = WSRealm()
HWIOS.plasmoids = Plasmoids()
site.addHandler("/ws", WebSocketRouter)
site.displayTracebacks = False
#override hwios general ssl setting
if self.hwios_config.has_option('general','ssl'):
from twisted.internet import ssl
from hwios.core.connection import ServerContextFactory
self.__service = SSLServer(self.config.getint('service', 'port'),site,ServerContextFactory())
else:
if self.config.getboolean('service','ssl'):
from twisted.internet import ssl
from hwios.core.connection import ServerContextFactory
self.__service = SSLServer(self.config.getint('service', 'port'),site,ServerContextFactory())
else: self.__service = TCPServer(self.config.getint('service','port'),site,100,self.config.get('service','listen'))
if os.getuid() == 0:
reactor.addSystemEventTrigger('after', 'startup', self.drop_privileges)
def __init__(self, application, port=80, resources=None, services=None, loud=False):
service.MultiService.__init__(self)
# Create, start and add a thread pool service, which is made available
# to our WSGIResource within HendrixResource
threads = ThreadPool()
reactor.addSystemEventTrigger('after', 'shutdown', threads.stop)
ThreadPoolService(threads).setServiceParent(self)
# create the base resource and add any additional static resources
resource = HendrixResource(reactor, threads, application, loud=loud)
if resources:
for res in resources:
resource.putNamedChild(res)
factory = server.Site(resource)
# add a tcp server that binds to port=port
main_web_tcp = TCPServer(port, factory)
main_web_tcp.setName('main_web_tcp') # to get this at runtime use hedrix_service.getServiceNamed('main_web_tcp')
main_web_tcp.setServiceParent(self)
# add any additional services
if services:
logger.info('loaded %r at %r'%(srv,srv_name))
for srv_name, srv in services:
srv.setName(srv_name)
srv.setServiceParent(self)
def configure(protocol, port, pipes, interface):
remove_all()
reactor.addSystemEventTrigger('after', 'shutdown', remove_all)
# gets default (outward-facing) network interface (e.g. deciding which of
# eth0, eth1, wlan0 is being used by the system to connect to the internet)
if interface == "auto":
interface = netifaces.gateways()['default'][netifaces.AF_INET][1]
else:
if interface not in netifaces.interfaces():
raise ValueError("Given interface does not exist.", interface)
add(protocol, port, interface)
manager = libnetfilter_queue.Manager()
def on_up(packet):
def accept():
manager.set_verdict(packet, libnetfilter_queue.NF_ACCEPT)
pipes.up.attempt(accept, packet.size)
def on_down(packet):
def accept():
manager.set_verdict(packet, libnetfilter_queue.NF_ACCEPT)
pipes.down.attempt(accept, packet.size)
manager.bind(UP_QUEUE, on_up)
manager.bind(DOWN_QUEUE, on_down)
reader = abstract.FileDescriptor()
reader.doRead = manager.process
reader.fileno = lambda: manager.fileno
reactor.addReader(reader)
def start():
# Import the module containing the resources
import sse
pool = threadpool.ThreadPool()
pool.start()
# Prepare the app root and subscribe endpoint
root = sse.Root()
subscribe = sse.Subscribe()
broadcast = sse.Broadcast(subscribe.publish_to_all)
connections = sse.Connections(subscribe.get_subscribers_count)
# Add sse and connections as children of root
root.putChild('sse', subscribe)
root.putChild('connections', connections)
root.putChild('broadcast', broadcast)
# Allow Ctrl-C to get you out cleanly:
reactor.addSystemEventTrigger('after', 'shutdown', pool.stop)
# emit an event every second
l = task.LoopingCall(publish_timestamp, { "broadcast": subscribe.publish_to_all })
l.start(1.0)
site = twisted_server.Site(root)
server = internet.TCPServer(1942, site)
application = service.Application("twisted-sse")
server.setServiceParent(application)
return application
def stop(self):
# stop the reactor
key = self.getKey()
reactor.addSystemEventTrigger('after', 'shutdown',
self._stopIterating, True, key)
reactor.stop()
self.iterate(key)
def shutdown(killPostProcessors = False, message = None):
""" Turn the knob that tells all parts of the program we're shutting down, optionally kill
any sub processes (that could prevent the program from exiting) and kill the twisted
reactor """
if Hellanzb.SHUTDOWN:
# shutdown already triggered
return
# that knob, that threads (PostProcessors) will check on before doing significant work
Hellanzb.SHUTDOWN = True
if killPostProcessors:
# However PostProcessors may be running sub-processes, which are all kill -9ed
# here
Topen.killAll()
if not getattr(Hellanzb, 'shutdownMessage', None):
Hellanzb.shutdownMessage = message
# stop the twisted reactor
if reactor.running:
# hellanzb downloader processes will call finishShutdown after reactor.run has
# completed (it has to: because the system event trigger below does NOT ensure
# finishShutdown is called in the final reactor iteration)
if not Hellanzb.IS_DOWNLOADER:
reactor.addSystemEventTrigger('after', 'shutdown', finishShutdown)
reactor.stop()
else:
finishShutdown()
def __init__(self, interface='', status_port=0, device_target=''):
self.interface = interface
self.device_target = device_target
self.status_port = status_port
self.port = reactor.listenMulticast(SSDP_PORT, self, listenMultiple=True) # pylint: disable=no-member
self.port.joinGroup(SSDP_ADDR, interface=interface)
reactor.addSystemEventTrigger('before', 'shutdown', self.stop) # pylint: disable=no-member
def main(self):
"""
Start the main event loop.
"""
if self.options.cycle:
reactor.callLater(0, self.heartbeat)
self.log.debug("Creating async MetricReporter")
self._metric_manager.start()
reactor.addSystemEventTrigger(
'before', 'shutdown', self._metric_manager.stop
)
# preserve legacy API
self.metricreporter = self._metric_manager.metricreporter
# Start ZenHub services
self._service_manager.start(self.dmd, reactor)
self._service_manager.onExecute(self.__workerItemCounter)
# Start Processing Invalidations
self.process_invalidations_task = task.LoopingCall(
self._invalidation_manager.process_invalidations
)
self.process_invalidations_task.start(
self.options.invalidation_poll_interval
)
reactor.run()
self.shutdown = True
getUtility(IEventPublisher).close()
if self.options.profiling:
self.profiler.stop()
def __init__(self, fd=42):
self.pid = os.getpid()
with os.fdopen(fd, 'r') as f:
self.cfg = json.loads(f.read())
self._log = os.fdopen(0, 'w', 1).write
set_proc_title(self.name)
set_pdeathsig(signal.SIGTERM)
def _sigusr1(SIG, FRM):
self.sigusr1()
signal.signal(signal.SIGUSR1, _sigusr1)
def _sigusr2(SIG, FRM):
self.sigusr2()
signal.signal(signal.SIGUSR2, _sigusr2)
def excepthook(*exc_info):
self.log("".join(traceback.format_exception(*exc_info)))
sys.excepthook = excepthook
reactor.addSystemEventTrigger("before", "shutdown", self.shutdown)
def __init__(self, name, behavior='delete', ttl=None, heartbeat_interval=None, lock_delay=None, host=CONSUL_HOST,
port=CONSUL_PORT, token=CONSUL_TOKEN, scheme=CONSUL_SCHEME, dc=CONSUL_DC, verify=CONSUL_VERIFY,
**kwargs):
"""
:type behavior: str
:param behavior: consul session behavior (release, delete)
:type ttl: int
:param ttl: time to live for the session before it is invalidated
:param name: session name to use
:type name: str
:param heartbeat_interval: interval (in seconds) in which a session
should be renewed, this value is also used as the session ttl.
:type heartbeat_interval: str
:type lock_delay: int
:param lock_delay: consul lock delay to use for sessions
"""
assert behavior in ('release', 'delete')
self.name = name
self.ttl = ttl or self.SESSION_TTL_SECONDS
self.heartbeat_interval = heartbeat_interval or self.SESSION_HEARTBEAT_SECONDS
self.lock_delay = lock_delay or self.SESSION_LOCK_DELAY_SECONDS
if 0 > self.lock_delay > 60:
self.logger.debug('invalid lock-delay=%s specified, using defaults', self.lock_delay)
self.lock_delay = 15
self.consul = Consul(host=host, port=port, token=token, scheme=scheme, dc=dc, verify=verify, **kwargs)
self.session = None
self.heartbeat = task.LoopingCall(self.session_renew)
reactor.callLater(0, self.session_create)
self.start()
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.addSystemEventTrigger('before', 'shutdown', self.session_destroy)
def main(args):
global start_time
if not args.host:
logger.info('No readers specified.')
return 0
# special case default Tari values
if args.modulation in Modulation_DefaultTari:
t_suggested = Modulation_DefaultTari[args.modulation]
if args.tari:
logger.warn('recommended Tari for %s is %d', args.modulation,
t_suggested)
else:
args.tari = t_suggested
logger.info('selected recommended Tari of %d for %s', args.tari,
args.modulation)
enabled_antennas = map(lambda x: int(x.strip()), args.antennas.split(','))
# d.callback will be called when all connections have terminated normally.
# use d.addCallback(<callable>) to define end-of-program behavior.
d = defer.Deferred()
d.addCallback(finish)
fac = LLRPClientFactory(onFinish=d,
duration=args.time,
report_every_n_tags=args.every_n,
antennas=enabled_antennas,
tx_power=args.tx_power,
modulation=args.modulation,
tari=args.tari,
session=args.session,
mode_index=args.mode_index,
mode_identifier=args.mode_identifier,
tag_population=args.population,
start_inventory=True,
disconnect_when_done=(args.time > 0),
reconnect=args.reconnect,
tag_content_selector={
'EnableROSpecID': False,
'EnableSpecIndex': False,
'EnableInventoryParameterSpecID': False,
'EnableAntennaID': True,
'EnableChannelIndex': False,
'EnablePeakRRSI': True,
'EnableFirstSeenTimestamp': False,
'EnableLastSeenTimestamp': True,
'EnableTagSeenCount': True,
'EnableAccessSpecID': False
})
# tag_report_cb will be called every time the reader sends a TagReport
# message (i.e., when it has "seen" tags).
fac.addTagReportCallback(tag_report_cb)
for host in args.host:
if ':' in host:
host, port = host.split(':', 1)
port = int(port)
else:
port = args.port
reactor.connectTCP(host, port, fac, timeout=3)
# catch ctrl-C and stop inventory before disconnecting
reactor.addSystemEventTrigger('before', 'shutdown', shutdown, fac)
# start runtime measurement to determine rates
start_time = time.time()
reactor.run()
def __init__(
self,
listen_interface=None,
outgoing_interface=None,
interface=None,
port=None,
standalone=False,
read_only_config_keys=None,
):
"""
Args:
listen_interface (str, optional): The IP address to listen to
BitTorrent connections on.
outgoing_interface (str, optional): The network interface name or
IP address to open outgoing BitTorrent connections on.
interface (str, optional): The IP address the daemon will
listen for UI connections on.
port (int, optional): The port the daemon will listen for UI
connections on.
standalone (bool, optional): If True the client is in Standalone
mode otherwise, if False, start the daemon as separate process.
read_only_config_keys (list of str, optional): A list of config
keys that will not be altered by core.set_config() RPC method.
"""
self.standalone = standalone
self.pid_file = get_config_dir('deluged.pid')
log.info('Deluge daemon %s', get_version())
if is_daemon_running(self.pid_file):
raise DaemonRunningError(
'Deluge daemon already running with this config directory!')
# Twisted catches signals to terminate, so just have it call the shutdown method.
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
# Catch some Windows specific signals
if windows_check():
def win_handler(ctrl_type):
"""Handle the Windows shutdown or close events."""
log.debug('windows handler ctrl_type: %s', ctrl_type)
if ctrl_type == CTRL_CLOSE_EVENT or ctrl_type == CTRL_SHUTDOWN_EVENT:
self._shutdown()
return 1
SetConsoleCtrlHandler(win_handler)
# Start the core as a thread and join it until it's done
self.core = Core(
listen_interface=listen_interface,
outgoing_interface=outgoing_interface,
read_only_config_keys=read_only_config_keys,
)
if port is None:
port = self.core.config['daemon_port']
self.port = port
if interface and not is_ip(interface):
log.error('Invalid UI interface (must be IP Address): %s',
interface)
interface = None
self.rpcserver = RPCServer(
port=port,
allow_remote=self.core.config['allow_remote'],
listen=not standalone,
interface=interface,
)
log.debug(
'Listening to UI on: %s:%s and bittorrent on: %s Making connections out on: %s',
interface,
port,
listen_interface,
outgoing_interface,
)
def __init__(self, daemon=False):
""" Init hub
@param daemon : True : launch as a daemon
"""
self.is_daemon = daemon
print(u"Domogik xPL Hub (python) v%s" % VERSION)
#print(u"Domogik xPL Hub (python) v%s" % 666)
print(u"Starting...")
print(u"- Reading configuration...")
### Read hub options
# read config file
config_p = ConfigParser.ConfigParser()
try:
with open(CONFIG_FILE) as cfg_file:
config_p.readfp(cfg_file)
# get config
config = {}
for k, v in config_p.items('hub'):
config[k] = v
except:
print(u"ERROR : Unable to open configuration file '%s' : %s" %
(CONFIG_FILE, traceback.format_exc()))
return
### Initiate the logger
print(u"- Preparing the log files...")
self.log = Logger(config['log_level'])
file_stdout = "%s/xplhub.log" % config['log_dir_path']
log.startLogging(open(file_stdout, "w"), setStdout=False)
self.log.info(u"------------------------")
file_clients = "%s/client_list.txt" % config['log_dir_path']
do_log_bandwidth = config['log_bandwidth']
if do_log_bandwidth == "True":
do_log_bandwidth = True
else:
do_log_bandwidth = False
file_bandwidth = "%s/bandwidth.csv" % config['log_dir_path']
do_log_invalid_data = config['log_invalid_data']
if do_log_invalid_data == "True":
do_log_invalid_data = True
else:
do_log_invalid_data = False
file_invalid_data = "%s/invalid_data.csv" % config['log_dir_path']
interfaces_list = config['interfaces']
allowed_interfaces = interfaces_list.replace(" ", "").split(",")
### Start listening to udp
print(u"- Initiating the multicast UDP client...")
self.log.info(u"Initiating the multicast UDP client...")
self.log.info(u"- creation...")
try:
self.MPP = UdpHub(log=self.log,
allowed_interfaces=allowed_interfaces,
file_clients=file_clients,
do_log_bandwidth=do_log_bandwidth,
file_bandwidth=file_bandwidth,
do_log_invalid_data=do_log_invalid_data,
file_invalid_data=file_invalid_data)
self.log.info(u"- start reactor...")
xpl_port = reactor.listenMulticast(3865, self.MPP) #,
#listenMultiple=True)
except:
msg = u"Error while starting the hub reactor. Exiting. Error : {0}".format(
traceback.format_exc())
self.log.error(msg)
print(msg)
self.stop_hub()
return
self.log.info(u"- add triggers...")
reactor.addSystemEventTrigger('during', 'shutdown', self.stop_hub)
print(u"xPL hub started!")
# following printed lines are in the xplhub.log file
self.log.info(u"xPL hub started")
reactor.run()
signal.signal(signal.SIGINT, SIGINT_CustomEventHandler)
try:
signal.signal(signal.SIGHUP, SIGHUP_CustomEventHandler)
except:
pass
signal.signal(signal.SIGTERM, SIGINT_CustomEventHandler)
signal.signal(signal.SIGABRT, SIGINT_CustomEventHandler)
# Make a XML-RPC Server listening to port
reactor.listenTCP(port, server.Site(app))
#Set up the loop to check the end of round every hour
endRoundLoop = task.LoopingCall(anwserver.endRoundCounter, app)
endRoundLoop.start(3600, 0)
#add a shutdown trigger to save all galaxies
reactor.addSystemEventTrigger('during', 'shutdown', anwserver.saveAllGalaxies, app)
if testemail:
emailResult = Services.inject(Email).sendTestEmail()
if emailResult == False:
logging.error("Email is enabled but is not working. Please check your configuration")
logHelp()
sys.exit(0)
# Start reactor
reactor.run()
def main(args):
if args.pid_file:
fp = file(args.pid_file, 'w')
fp.write(str(os.getpid()))
fp.close()
if args.port != 3333:
'''User most likely provided host/port
for getwork interface. Let's try to detect
Stratum host/port of given getwork pool.'''
try:
new_host = (yield utils.detect_stratum(args.host, args.port))
except:
log.exception("Stratum host/port autodetection failed")
new_host = None
if new_host != None:
args.host = new_host[0]
args.port = new_host[1]
log.warning("Stratum proxy version: %s" % version.VERSION)
# Setup periodic checks for a new version
#test_update()
if args.tor:
log.warning("Configuring Tor connection")
args.proxy = '127.0.0.1:9050'
args.host = 'pool57wkuu5yuhzb.onion'
args.port = 3333
if args.proxy:
proxy = args.proxy.split(':')
if len(proxy) < 2:
proxy = (proxy, 9050)
else:
proxy = (proxy[0], int(proxy[1]))
log.warning("Using proxy %s:%d" % proxy)
else:
proxy = None
log.warning("Trying to connect to Stratum pool at %s:%d" %
(args.host, args.port))
# Connect to Stratum pool
f = SocketTransportClientFactory(
args.host,
args.port,
debug=args.verbose,
proxy=proxy,
event_handler=client_service.ClientMiningService)
job_registry = jobs.JobRegistry(f,
cmd=args.blocknotify_cmd,
scrypt_target=args.scrypt_target,
no_midstate=args.no_midstate,
real_target=args.real_target,
use_old_target=args.old_target)
client_service.ClientMiningService.job_registry = job_registry
client_service.ClientMiningService.reset_timeout()
workers = worker_registry.WorkerRegistry(f)
f.on_connect.addCallback(on_connect, workers, job_registry)
f.on_disconnect.addCallback(on_disconnect, workers, job_registry)
if args.test:
f.on_connect.addCallback(test_launcher, job_registry)
# Cleanup properly on shutdown
reactor.addSystemEventTrigger('before', 'shutdown', on_shutdown, f)
# Block until proxy connect to the pool
yield f.on_connect
# Setup getwork listener
if args.getwork_port > 0:
conn = reactor.listenTCP(
args.getwork_port,
Site(
getwork_listener.Root(job_registry,
workers,
stratum_host=args.stratum_host,
stratum_port=args.stratum_port,
custom_lp=args.custom_lp,
custom_stratum=args.custom_stratum,
custom_user=args.custom_user,
custom_password=args.custom_password)),
interface=args.getwork_host)
try:
conn.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE,
1) # Enable keepalive packets
conn.socket.setsockopt(
socket.SOL_TCP, socket.TCP_KEEPIDLE,
60) # Seconds before sending keepalive probes
conn.socket.setsockopt(
socket.SOL_TCP, socket.TCP_KEEPINTVL,
1) # Interval in seconds between keepalive probes
conn.socket.setsockopt(
socket.SOL_TCP, socket.TCP_KEEPCNT,
5) # Failed keepalive probles before declaring other end dead
except:
pass # Some socket features are not available on all platforms (you can guess which one)
# Setup stratum listener
#if args.stratum_port > 0:
# stratum_listener.StratumProxyService._set_upstream_factory(f)
# stratum_listener.StratumProxyService._set_custom_user(args.custom_user, args.custom_password)
# reactor.listenTCP(args.stratum_port, SocketTransportFactory(debug=False, event_handler=ServiceEventHandler), interface=args.stratum_host)
# Setup multicast responder
#reactor.listenMulticast(3333, multicast_responder.MulticastResponder((args.host, args.port), args.stratum_port, args.getwork_port), listenMultiple=True)
log.warning(
"-----------------------------------------------------------------------"
)
if args.getwork_host == '0.0.0.0':
log.warning("PROXY IS LISTENING ON ALL IPs ON PORT %d (getwork)" %
args.getwork_port)
else:
log.warning("LISTENING FOR MINERS ON http://%s:%d (getwork) " % \
(args.getwork_host, args.getwork_port))
log.warning(
"-----------------------------------------------------------------------"
)
"--endpoint",
type=str,
default=DEFAULT_ZMQ_SUBSCRIBE_ENDPOINT,
help="endpoint to subscribe to for zmq notifications from "
"c-lightning via cl-zmq.py plugin")
parser.add_argument("-m",
"--mock-endpoint",
type=str,
default=DEFAULT_MOCK_ZMQ_SUBSCRIBE_ENDPOINT,
help="endpoint to subscribe to zmq notifcations from a "
"test script such as mock-png.py")
parser.add_argument("-w",
"--websocket-port",
type=int,
default=DEFAULT_WEBSOCKET_PORT,
help="port to listen for incoming websocket connections")
parser.add_argument("-a",
"--art-db-dir",
type=str,
default=DEFAULT_ART_DB_DIR,
help="directory to save the image state and logs")
settings = parser.parse_args()
a = App(settings.endpoint, settings.mock_endpoint, settings.websocket_port,
settings.art_db_dir)
a.run()
reactor.addSystemEventTrigger("before", "shutdown", a.stop)
reactor.run()
def getSession():
global _session
if _session is None:
_session = txrequests.Session()
reactor.addSystemEventTrigger("before", "shutdown", closeSession)
return _session
def react(main, argv=(), _reactor=None):
"""
Call C{main} and run the reactor until the L{Deferred} it returns fires or
the coroutine it returns completes.
This is intended as the way to start up an application with a well-defined
completion condition. Use it to write clients or one-off asynchronous
operations. Prefer this to calling C{reactor.run} directly, as this
function will also:
- Take care to call C{reactor.stop} once and only once, and at the right
time.
- Log any failures from the C{Deferred} returned by C{main}.
- Exit the application when done, with exit code 0 in case of success and
1 in case of failure. If C{main} fails with a C{SystemExit} error, the
code returned is used.
The following demonstrates the signature of a C{main} function which can be
used with L{react}::
async def main(reactor, username, password):
return "ok"
task.react(main, ("alice", "secret"))
@param main: A callable which returns a L{Deferred} or
coroutine. It should take the reactor as its first
parameter, followed by the elements of C{argv}.
@param argv: A list of arguments to pass to C{main}. If omitted the
callable will be invoked with no additional arguments.
@param _reactor: An implementation detail to allow easier unit testing. Do
not supply this parameter.
@since: 12.3
"""
if _reactor is None:
from twisted.internet import reactor as _reactor
finished = defer.ensureDeferred(main(_reactor, *argv))
codes = [0]
stopping = []
_reactor.addSystemEventTrigger("before", "shutdown", stopping.append, True)
def stop(result, stopReactor):
if stopReactor:
try:
_reactor.stop()
except ReactorNotRunning:
pass
if isinstance(result, Failure):
if result.check(SystemExit) is not None:
code = result.value.code
else:
log.err(result, "main function encountered error")
code = 1
codes[0] = code
def cbFinish(result):
if stopping:
stop(result, False)
else:
_reactor.callWhenRunning(stop, result, True)
finished.addBoth(cbFinish)
_reactor.run()
sys.exit(codes[0])
def main():
# This is necessary for testing with non-HTTPS localhost
# Remove this if deploying to production
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
# This is necessary because Azure does not guarantee
# to return scopes in the same case and order as requested
os.environ['OAUTHLIB_RELAX_TOKEN_SCOPE'] = '1'
os.environ['OAUTHLIB_IGNORE_SCOPE_CHANGE'] = '1'
logging.basicConfig(level=logging.DEBUG, filemode='w')
parser = argparse.ArgumentParser(prog='oauthsub',
description=auth_service.__doc__)
config_dict = configuration.Configuration().serialize()
setup_parser(parser, config_dict)
args = parser.parse_args()
if args.dump_config:
dump_config(configuration.Configuration(), sys.stdout)
sys.exit(0)
if args.config_file:
configpath = os.path.expanduser(args.config_file)
config_dict["__file__"] = os.path.realpath(configpath)
with io.open(configpath, 'r', encoding='utf8') as infile:
# pylint: disable=W0122
exec(infile.read(), config_dict)
config_dict.pop("__file__")
for key, value in vars(args).items():
if key in config_dict and value is not None:
config_dict[key] = value
config = configuration.Configuration(**config_dict)
# Create directory for logs if it doesn't exist
if not os.path.exists(config.logdir):
os.makedirs(config.logdir)
# We'll add a handler which puts log events in an actual file for review as
# needed. We'll put the log file on a rotation where each log may grow up to
# 1 megabyte with up to 10 backups
filelog = logging.handlers.RotatingFileHandler(os.path.join(
config.logdir, 'oauthsub.log'),
maxBytes=int(1e6),
backupCount=10)
# We'll add a timestamp to the format for this log
format_str = ('%(asctime)s %(levelname)-4s %(filename)s [%(lineno)-3s] :'
' %(message)s')
filelog.setFormatter(logging.Formatter(format_str))
logging.getLogger("").addHandler(filelog)
config_dict = config.serialize()
config_dict.pop('secrets', None)
config_dict.pop('client_secrets', None)
logging.info('Configuration: %s',
json.dumps(config_dict, indent=2, sort_keys=True))
# NOTE(josh): hack to deal with jinja's failure to resolve relative imports
# to absolute paths
oauthsub.__file__ = os.path.abspath(oauthsub.__file__)
app = auth_service.Application(config)
if config.server == "flask":
app.run(threaded=True, host=config.host, port=config.port)
elif config.server == "gevent":
from gevent.pywsgi import WSGIServer
WSGIServer((config.host, config.port), app).serve_forever()
elif config.server == "twisted":
from twisted.web import server
from twisted.web.wsgi import WSGIResource
from twisted.python.threadpool import ThreadPool
from twisted.internet import reactor
thread_pool = ThreadPool()
thread_pool.start()
reactor.addSystemEventTrigger('after', 'shutdown', thread_pool.stop)
resource = WSGIResource(reactor, thread_pool, app)
factory = server.Site(resource)
reactor.listenTCP(config.port, factory)
reactor.run()
def init_proto(proto):
self._connect_successes += 1
reactor.addSystemEventTrigger('before', 'shutdown', cleanup, proto)
return proto
'''
from syslogger import init_logger
syslog_conf = config.syslog_conf.split(',')
if len(syslog_conf) > 1:
syslog_conf = (syslog_conf[0], int(syslog_conf[1]))
else:
syslog_conf = syslog_conf[0]
init_logger( syslog_conf )
'''
from server import Server
from datetime import datetime
server = Server()
reactor.addSystemEventTrigger('before', 'shutdown', server.cleanup)
application = service.Application(SERVER_NAME)
log_path = config.log_path
log_rotate = int(config.log_rotate_interval)
logfile = LogFile('message.log', log_path, rotateLength=log_rotate)
logOb = FileLogObserver(logfile)
logOb.formatTime = lambda when: datetime.fromtimestamp(when).strftime(
'%m/%d %T.%f')
application.setComponent(ILogObserver, logOb.emit)
internet.TCPServer(config.port, server,
interface=config.interface).setServiceParent(
service.IServiceCollection(application))
internet.TCPServer(config.adminport,
def main(reactor):
reactor.callLater(1, reactor.stop)
finished = defer.Deferred()
reactor.addSystemEventTrigger('during', 'shutdown',
finished.callback, None)
return finished
self.sendPacket(res)
elif sdl2.SDL_GetKeyName(
event.key.keysym.sym).lower() == b'd':
res = ChangePacket(1)
print("Changed camera to 1")
self.sendPacket(res)
frames.pop(packet.frame_idx)
res = AckPacket(packet.frame_idx, interarrival_time)
self.sendPacket(res)
elif packet.type == PacketType.RESPONSE:
self.previous_packet_time = get_us()
elif packet.type == PacketType.FINISH:
reactor.stop()
fn = datetime.now().isoformat().replace(":", "-")
f = open(f"videos/{fn}.264", "wb")
h264 = av.Codec("h264")
h264_decoder = av.CodecContext.create(h264)
sdl2.ext.init()
window = sdl2.ext.Window("frc-usb-camera", size=(1280, 720))
window.show()
window_surface = sdl2.SDL_GetWindowSurface(window.window)
window_array = sdl2.ext.pixels3d(window_surface.contents)
vc = VideoClient()
reactor.listenUDP(2601, vc)
reactor.addSystemEventTrigger('before', 'shutdown', vc.finishConnection)
reactor.run()
def start(self):
reactor.addSystemEventTrigger('during', 'shutdown', self.stop)
ThreadPool.start(self)
def _start(self):
log.info('starting zenactiond consumer.')
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
self._consumer.run()
def __init__(self):
# command line parsing utils
self.parser = OptionParser()
self.cmdline_options = None
# version
self.version_string = __version__
# testing
# This variable is to be able to hook/bypass code when unit-tests are runned
self.testing = False
# daemon
self.nodaemon = False
# thread pool size of 1
self.orm_tp = ThreadPool(0, 1)
self.bind_addresses = '127.0.0.1'
# bind port
self.bind_port = 8082
# store name
self.store_name = 'main_store'
self.db_type = 'sqlite'
# debug defaults
self.orm_debug = False
self.log_requests_responses = -1
self.requests_counter = 0
self.loglevel = "CRITICAL"
# files and paths
self.root_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
self.pid_path = '/var/run/globaleaks'
self.working_path = '/var/globaleaks'
self.static_source = '/usr/share/globaleaks/data'
self.client_path = '/usr/share/globaleaks/client'
for path in possible_client_paths:
if os.path.exists(path):
self.client_path = path
break
self.set_ramdisk_path()
self.default_password = '******'
# some singleton classes: sessions and some event queues
self.authentication_lifetime = 3600
self.sessions = TempDict(timeout=self.authentication_lifetime)
self.RecentEventQ = []
self.RecentAnomaliesQ = {}
self.accept_submissions = True
# statistical, referred to latest period
# and resetted by session_management sched
self.failed_login_attempts = 0
# static file rules
self.staticfile_regexp = r'(.*)'
self.staticfile_overwrite = False
self.reserved_names = OD({
'logo': 'logo',
'css': 'custom_stylesheet',
'html': 'custom_homepage'
})
# acceptable 'Host:' header in HTTP request
self.accepted_hosts = "127.0.0.1, localhost"
self.configured_hosts = []
self.receipt_regexp = u'[0-9]{16}'
# A lot of operations performed massively by globaleaks
# should avoid to fetch continuously variables from the DB so that
# it is important to keep this variables in memory
#
# The following initialization is needed only for variables that need
# to be used in the startup queries, after that memory_copy is
# initialized with the content Node table.
self.memory_copy = OD({
'maximum_namesize': 128,
'maximum_textsize': 4096,
'maximum_filesize': 30,
'allow_iframes_inclusion': False,
'tor2web_access': {
'admin': True,
'whistleblower': False,
'custodian': False,
'receiver': False,
'unauth': True
}
})
# Default request time uniform value
self.side_channels_guard = 0.150
# unchecked_tor_input contains information that cannot be validated now
# due to complex inclusions or requirements. Data is used in
# globaleaks.db.appdata.apply_cli_options()
self.unchecked_tor_input = {}
# SOCKS default
self.socks_host = "127.0.0.1"
self.socks_port = 9050
self.notification_limit = 30
self.jobs_operation_limit = 20
self.user = getpass.getuser()
self.group = getpass.getuser()
self.uid = os.getuid()
self.gid = os.getgid()
self.start_clean = False
self.devel_mode = False
self.developer_name = ''
self.skip_wizard = False
self.log_timing_stats = False
# Number of failed login enough to generate an alarm
self.failed_login_alarm = 5
# Number of minutes in which a user is prevented to login in case of triggered alarm
self.failed_login_block_time = 5
# Alarm to be ignored: can be raise with the -A command line switch
self.disk_alarm_threshold = 0
# Size in bytes of every log file. Once this size is reached the
# logfile is rotated.
# Default: 1M
self.log_file_size = 1000000
# Number of log files to conserve.
self.maximum_rotated_log_files = 100
# size used while streaming files
self.file_chunk_size = 8192
self.AES_key_size = 32
self.AES_key_id_regexp = u'[A-Za-z0-9]{16}'
self.AES_counter_nonce = 128 / 8
self.AES_file_regexp = r'(.*)\.aes'
self.AES_file_regexp_comp = re.compile(self.AES_file_regexp)
self.AES_keyfile_prefix = "aeskey-"
self.exceptions = {}
self.exceptions_email_count = 0
self.exceptions_email_hourly_limit = 20
# Extreme debug options triggered by --XXX, that's are the defaults
self.debug_option_in_the_future = 0
self.debug_option_UUID_human = ""
self.debug_UUID_human_counter = 0
self.debug_option_mlockall = False
self.disable_mail_torification = False
self.disable_mail_notification = False
self.disable_backend_exception_notification = False
self.disable_client_exception_notification = False
self.enable_input_length_checks = True
self.mail_counters = {}
self.mail_timeout = 15 # seconds
self.mail_attempts_limit = 3 # per mail limit
reactor.addSystemEventTrigger('after', 'shutdown', self.orm_tp.stop)
self.orm_tp.start()
def main(mode, iface):
klass = Server if mode == 'server' else Client
obj = klass(iface)
reactor.addSystemEventTrigger('before', 'shutdown', obj.stop)
def main():
reactor.disconnectAll()
failover = False
if settings.POOL_FAILOVER_ENABLE:
failover = settings.failover_pool
settings.failover_pool = not settings.failover_pool
pool_host = settings.POOL_HOST
pool_port = settings.POOL_PORT
if failover and settings.POOL_FAILOVER_ENABLE:
pool_host = settings.POOL_HOST_FAILOVER
pool_port = settings.POOL_PORT_FAILOVER
log.warning("Monero Stratum proxy version: %s" % version.VERSION)
log.warning("Trying to connect to Stratum pool at %s:%d" %
(pool_host, pool_port))
# Connect to Stratum pool, main monitoring connection
f = SocketTransportClientFactory(
pool_host,
pool_port,
debug=settings.DEBUG,
proxy=None,
event_handler=client_service.ClientMiningService)
job_registry = jobs.JobRegistry(f)
client_service.ClientMiningService.job_registry = job_registry
client_service.ClientMiningService.reset_timeout()
f.on_connect.addCallback(on_connect)
f.on_disconnect.addCallback(on_disconnect)
# Cleanup properly on shutdown
reactor.addSystemEventTrigger('before', 'shutdown', on_shutdown, f)
# Block until proxy connect to the pool
try:
yield f.on_connect
except TransportException:
log.warning(
"First pool server must be online first time to start failover")
return
# Setup stratum listener
stratum_listener.StratumProxyService._set_upstream_factory(f)
stratum_listener.StratumProxyService._set_custom_user(
settings.CUSTOM_USER, settings.CUSTOM_PASSWORD,
settings.ENABLE_WORKER_ID, settings.WORKER_ID_FROM_IP)
reactor.listenTCP(settings.STRATUM_PORT,
SocketTransportFactory(
debug=settings.DEBUG,
event_handler=ServiceEventHandler),
interface=settings.STRATUM_HOST)
# Setup multicast responder
reactor.listenMulticast(3333,
multicast_responder.MulticastResponder(
(pool_host, pool_port), settings.STRATUM_PORT),
listenMultiple=True)
log.warning(
"-----------------------------------------------------------------------"
)
if settings.STRATUM_HOST == '0.0.0.0':
log.warning("PROXY IS LISTENING ON ALL IPs ON PORT %d (stratum)" %
settings.STRATUM_PORT)
else:
log.warning("LISTENING FOR MINERS ON stratum+tcp://%s:%d (stratum)" % \
(settings.STRATUM_HOST, settings.STRATUM_PORT))
log.warning(
"-----------------------------------------------------------------------"
)
.format(_times, queue.qsize()))
if loop:
reactor.callLater(SYNC_DB_INTERVAL, sync_dirty_attributes, queue)
else:
log.debug('End sync db, dirty attributes length {0}, loop:{1}'.format(
queue.qsize(), loop))
try:
DIRTY_ATTRIBUTES
except NameError:
DIRTY_ATTRIBUTES = Queue.Queue()
reactor.callLater(SYNC_DB_INTERVAL, sync_dirty_attributes,
DIRTY_ATTRIBUTES)
reactor.addSystemEventTrigger('before', 'shutdown', sync_dirty_attributes,
DIRTY_ATTRIBUTES, False)
class AttributeError(Exception):
pass
class Attribute(object): #Present one row in db
def __init__(self, name):
self.__name = name
self.__uid = 0
self.__new = False
self.__del = False
self.__dirty = False
self.__dirty_fields = []
def run(self):
"""Starts a polling job for netbox.
:returns: A Deferred, whose result will be True when the job did
something, or False when the job did nothing (i.e. no
plugins ran).
"""
self.netbox = yield db.run_in_thread(dataloader.load_netbox,
self.netbox_id)
self._log_context.update(dict(job=self.name,
sysname=self.netbox.sysname))
self._logger.debug("Job %r started with plugins: %r",
self.name, self.plugins)
# Initialize netbox in container
nb = self._container_factory(shadows.Netbox, key=None,
id=self.netbox.id,
sysname=self.netbox.sysname)
self._create_agentproxy()
plugins = yield self._find_plugins()
self._reset_timers()
if not plugins:
self._destroy_agentproxy()
defer.returnValue(False)
self._logger.debug("Starting job %r for %s",
self.name, self.netbox.sysname)
def wrap_up_job(_result):
self._logger.debug("Job %s for %s done.", self.name,
self.netbox.sysname)
self._log_timings()
return True
def plugin_failure(failure):
self._log_timings()
if not failure.check(AbortedJobError):
raise AbortedJobError("Job aborted due to plugin failure",
cause=failure.value)
return failure
def save_failure(failure):
if not failure.check(db.ResetDBConnectionError):
log_unhandled_failure(self._logger,
failure,
"Save stage failed with unhandled error")
self._log_timings()
raise AbortedJobError("Job aborted due to save failure",
cause=failure.value)
def log_abort(failure):
if failure.check(SuggestedReschedule):
return failure
if failure.check(AbortedJobError):
self._logger.error("Job %r for %s aborted: %s",
self.name, self.netbox.sysname,
failure.value)
return failure
def save(result):
if self.cancelled.isSet():
return wrap_up_job(result)
df = self._save_container()
df.addErrback(save_failure)
df.addCallback(wrap_up_job)
return df
# pylint is unable to find reactor members:
# pylint: disable=E1101
shutdown_trigger_id = reactor.addSystemEventTrigger(
"before", "shutdown", self.cancel)
def cleanup(result):
self._destroy_agentproxy()
reactor.removeSystemEventTrigger(shutdown_trigger_id)
return result
def log_externally_success(result):
self._log_job_externally(True if result else None)
return result
def log_externally_failure(result):
self._log_job_externally(False)
return result
# The action begins here
df = self._iterate_plugins(plugins)
df.addErrback(plugin_failure)
df.addCallback(save)
df.addErrback(log_abort)
df.addBoth(cleanup)
df.addCallbacks(log_externally_success, log_externally_failure)
yield df
defer.returnValue(True)
def __init__(
self,
host: str = None,
port: int = None,
username: str = None,
password: str = None,
options: dict = {},
callbacks: dict = {},
log_level: str = 'WARNING',
):
"""Initialize Monitor:
connection parameters: host, port, username, password,
options: a dict of connection parameters transmitted to the server in the 'auth' message
callbacks: {'channel': function ...}
where channel is one of CHANNELS
use '*' as a channel name to set a new default callback function
and function is the callback function that will receive (channel, message)
callbacks must return True to continue the monitor.run() loop
by default, all callbacks will print to stdout; to override this, pass callbacks={}
log_level: select filter for log: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'
"""
logging.basicConfig(
stream=sys.stderr,
level=log_level,
format=
"[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s"
)
logger = logging.getLogger(self.__class__.__name__)
logging.info(
f"{self} __init__({host}, {port}, {username}, XXXXXXXX, {options}, {callbacks}, {log_level})"
)
if not isinstance(options, dict):
raise ValueError(
f'options: expected dict type, got {type(options)}')
self.host = host or os.environ.get('TXTRADER_HOST',
DEFAULT_TXTRADER_HOST)
self.port = int(
port
or os.environ.get('TXTRADER_TCP_PORT', DEFAULT_TXTRADER_TCP_PORT))
self.username = username or os.environ.get('TXTRADER_USERNAME',
DEFAULT_TXTRADER_USERNAME)
self.password = password or os.environ.get('TXTRADER_PASSWORD',
DEFAULT_TXTRADER_PASSWORD)
self.options = options
# setup callback map
self.set_callbacks(callbacks)
# store LoopingCall if client calls set_tick_interval
self.tickers = set()
self.shutdown_pending = False
reactor.addSystemEventTrigger('after', 'startup', self.startup_event)
reactor.addSystemEventTrigger('before', 'shutdown',
self.shutdown_event)
# create the factory singleton
self.factory = StatusClientFactory(self)
self.connection_state = ConnectionState.INITIALIZING
self.connection_wanted = False
self.connection = None
self.connector = None
def run(self):
self.slot.schedule(on_start=True)
self._logging_task.start(30)
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
reactor.run()
print('[*] config.json is missing !')
exit()
if args.frida:
start_frida_script()
crypto = Crypto(config['ServerKey'])
replay = Replay(config['ReplayDirectory'])
client_endpoint = ClientEndpoint(reactor, config['Hostname'],
config['Port'])
server_endpoint = ServerEndpoint(reactor, config['Port'])
udp_protocol = UDPProtocol(config['UDPHost'], config['UDPPort'],
replay) if args.udp else None
server_endpoint.listen(
ServerFactory(client_endpoint, udp_protocol, crypto, replay, args))
print("[*] TCP Proxy is listening on {}:{}".format(
server_endpoint.interface, server_endpoint.port))
if udp_protocol is not None:
udp_listener = reactor.listenUDP(config['UDPPort'], udp_protocol)
udp_listener_host = udp_listener.getHost()
print("[*] UDP Proxy is listening on {}:{}".format(
udp_listener_host.host, udp_listener_host.port))
reactor.addSystemEventTrigger('before', 'shutdown', onClose, udp_protocol)
reactor.run()
def __init__(
self,
url,
baseDN,
credentials=None,
timeout=None,
tlsCACertificateFile=None,
tlsCACertificateDirectory=None,
useTLS=False,
fieldNameToAttributesMap=DEFAULT_FIELDNAME_ATTRIBUTE_MAP,
recordTypeSchemas=DEFAULT_RECORDTYPE_SCHEMAS,
extraFilters=None,
ownThreadpool=True,
threadPoolMax=10,
authConnectionMax=5,
queryConnectionMax=5,
tries=3,
warningThresholdSeconds=5,
_debug=False,
):
"""
@param url: The URL of the LDAP server to connect to.
@type url: L{unicode}
@param baseDN: The base DN for queries.
@type baseDN: L{unicode}
@param credentials: The credentials to use to authenticate with the
LDAP server.
@type credentials: L{IUsernamePassword}
@param timeout: A timeout, in seconds, for LDAP queries.
@type timeout: number
@param tlsCACertificateFile: ...
@type tlsCACertificateFile: L{FilePath}
@param tlsCACertificateDirectory: ...
@type tlsCACertificateDirectory: L{FilePath}
@param useTLS: Enable the use of TLS.
@type useTLS: L{bool}
@param fieldNameToAttributesMap: A mapping of field names to LDAP
attribute names.
@type fieldNameToAttributesMap: mapping with L{NamedConstant} keys and
sequence of L{unicode} values
@param recordTypeSchemas: Schema information for record types.
@type recordTypeSchemas: mapping from L{NamedConstant} to
L{RecordTypeSchema}
@param extraFilters: A dict (keyed off recordType) of extra filter
fragments to AND in to any generated queries.
@type extraFilters: L{dicts} of L{unicode}
"""
self.url = url
self._baseDN = baseDN
self._credentials = credentials
self._timeout = timeout
self._extraFilters = extraFilters
self._tries = tries
self._warningThresholdSeconds = warningThresholdSeconds
if tlsCACertificateFile is None:
self._tlsCACertificateFile = None
else:
self._tlsCACertificateFile = tlsCACertificateFile.path
if tlsCACertificateDirectory is None:
self._tlsCACertificateDirectory = None
else:
self._tlsCACertificateDirectory = tlsCACertificateDirectory.path
self._useTLS = useTLS
if _debug:
self._debug = 255
else:
self._debug = None
if self.fieldName.recordType in fieldNameToAttributesMap:
raise TypeError("Record type field may not be mapped")
if BaseFieldName.uid not in fieldNameToAttributesMap:
raise DirectoryConfigurationError("Mapping for uid required")
self._fieldNameToAttributesMap = fieldNameToAttributesMap
self._attributeToFieldNameMap = {}
for name, attributes in fieldNameToAttributesMap.iteritems():
for attribute in attributes:
if ":" in attribute:
attribute, ignored = attribute.split(":", 1)
self._attributeToFieldNameMap.setdefault(attribute,
[]).append(name)
self._recordTypeSchemas = recordTypeSchemas
attributesToFetch = set()
for attributes in fieldNameToAttributesMap.values():
for attribute in attributes:
if ":" in attribute:
attribute, ignored = attribute.split(":", 1)
attributesToFetch.add(attribute.encode("utf-8"))
self._attributesToFetch = list(attributesToFetch)
# Threaded connection pool.
# The connection size limit here is the size for connections doing
# queries.
# There will also be one-off connections for authentications which also
# run in their own threads.
# Thus the threadpool max ought to be larger than the connection max to
# allow for both pooled query connections and one-off auth-only
# connections.
self.ownThreadpool = ownThreadpool
if self.ownThreadpool:
self.threadpool = ThreadPool(
minthreads=1,
maxthreads=threadPoolMax,
name="LDAPDirectoryService",
)
else:
# Use the default threadpool but adjust its size to fit our needs
self.threadpool = reactor.getThreadPool()
self.threadpool.adjustPoolsize(
max(threadPoolMax, self.threadpool.max))
# Separate pools for LDAP queries and LDAP binds.
self.connectionPools = {
"query": ConnectionPool("query", self, credentials,
queryConnectionMax),
"auth": ConnectionPool("auth", self, None, authConnectionMax),
}
self.poolStats = collections.defaultdict(int)
reactor.callWhenRunning(self.start)
reactor.addSystemEventTrigger("during", "shutdown", self.stop)
if __name__ == "__main__":
debug = True if config.environment.lower() in ["dev", "development"
] else False
# Init the database pool
pool.init()
log.startLogging(sys.stdout)
wsgiResource = WSGIResource(reactor, reactor.getThreadPool(), app)
MatchaServerFactory = get_server_factory(app)
MatchaServerProtocol = get_server_protocol(app)
wsFactory = MatchaServerFactory("ws://0.0.0.0:5000")
wsFactory.protocol = MatchaServerProtocol
app.socks = wsFactory
wsResource = WebSocketResource(wsFactory)
rootResource = WSGIRootResource(wsgiResource, {b'ws': wsResource})
site = Site(rootResource)
reactor.addSystemEventTrigger('before', 'shutdown', shutdown_server)
# signal.signal(signal.SIGINT, signal.default_int_handler)
reactor.listenTCP(5000, site)
reactor.run()
def __init__(self, hs):
self.is_mine = hs.is_mine
self.is_mine_id = hs.is_mine_id
self.clock = hs.get_clock()
self.store = hs.get_datastore()
self.wheel_timer = WheelTimer()
self.notifier = hs.get_notifier()
self.replication = hs.get_replication_layer()
self.federation = hs.get_federation_sender()
self.state = hs.get_state_handler()
self.replication.register_edu_handler("m.presence",
self.incoming_presence)
self.replication.register_edu_handler(
"m.presence_invite", lambda origin, content: self.invite_presence(
observed_user=UserID.from_string(content["observed_user"]),
observer_user=UserID.from_string(content["observer_user"]),
))
self.replication.register_edu_handler(
"m.presence_accept", lambda origin, content: self.accept_presence(
observed_user=UserID.from_string(content["observed_user"]),
observer_user=UserID.from_string(content["observer_user"]),
))
self.replication.register_edu_handler(
"m.presence_deny", lambda origin, content: self.deny_presence(
observed_user=UserID.from_string(content["observed_user"]),
observer_user=UserID.from_string(content["observer_user"]),
))
distributor = hs.get_distributor()
distributor.observe("user_joined_room", self.user_joined_room)
active_presence = self.store.take_presence_startup_info()
# A dictionary of the current state of users. This is prefilled with
# non-offline presence from the DB. We should fetch from the DB if
# we can't find a users presence in here.
self.user_to_current_state = {
state.user_id: state
for state in active_presence
}
metrics.register_callback("user_to_current_state_size",
lambda: len(self.user_to_current_state))
now = self.clock.time_msec()
for state in active_presence:
self.wheel_timer.insert(
now=now,
obj=state.user_id,
then=state.last_active_ts + IDLE_TIMER,
)
self.wheel_timer.insert(
now=now,
obj=state.user_id,
then=state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT,
)
if self.is_mine_id(state.user_id):
self.wheel_timer.insert(
now=now,
obj=state.user_id,
then=state.last_federation_update_ts +
FEDERATION_PING_INTERVAL,
)
else:
self.wheel_timer.insert(
now=now,
obj=state.user_id,
then=state.last_federation_update_ts + FEDERATION_TIMEOUT,
)
# Set of users who have presence in the `user_to_current_state` that
# have not yet been persisted
self.unpersisted_users_changes = set()
reactor.addSystemEventTrigger("before", "shutdown", self._on_shutdown)
self.serial_to_user = {}
self._next_serial = 1
# Keeps track of the number of *ongoing* syncs on this process. While
# this is non zero a user will never go offline.
self.user_to_num_current_syncs = {}
# Keeps track of the number of *ongoing* syncs on other processes.
# While any sync is ongoing on another process the user will never
# go offline.
# Each process has a unique identifier and an update frequency. If
# no update is received from that process within the update period then
# we assume that all the sync requests on that process have stopped.
# Stored as a dict from process_id to set of user_id, and a dict of
# process_id to millisecond timestamp last updated.
self.external_process_to_current_syncs = {}
self.external_process_last_updated_ms = {}
# Start a LoopingCall in 30s that fires every 5s.
# The initial delay is to allow disconnected clients a chance to
# reconnect before we treat them as offline.
self.clock.call_later(
30,
self.clock.looping_call,
self._handle_timeouts,
5000,
)
self.clock.call_later(
60,
self.clock.looping_call,
self._persist_unpersisted_changes,
60 * 1000,
)
metrics.register_callback("wheel_timer_size",
lambda: len(self.wheel_timer))
def __init__(self):
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
self.market_feeds = [
get_default_market_feed(currency_pair)
for currency_pair in CURRENCY_PAIRS
]
def run(self):
## Add a callback for the startup/shutdown stuff
reactor.addSystemEventTrigger('before', 'startup', self.on_startup)
reactor.addSystemEventTrigger('before', 'shutdown', self.on_shutdown)
## And start reactor
reactor.run()
def main():
lg.set_debug_level(18)
lg.life_begins()
from crypt import key
key.InitMyKey()
from contacts import identitycache
identitycache.init()
from system import tmpfile
tmpfile.init()
from services import driver
driver.disabled_services().add('service_tcp_connections')
driver.disabled_services().add('service_p2p_hookups')
driver.disabled_services().add('service_nodes_lookup')
driver.disabled_services().add('service_identity_propagate')
driver.disabled_services().add('service_ip_port_responder')
driver.init()
driver.enabled_services().clear()
driver.enabled_services().add('service_udp_transport')
driver.enabled_services().add('service_udp_datagrams')
driver.enabled_services().add('service_my_ip_port')
driver.enabled_services().add('service_gateway')
driver.enabled_services().add('service_entangled_dht')
driver.enabled_services().add('service_network')
driver.start()
# options = { 'idurl': my_id.getIDURL(),}
# options['host'] = nameurl.GetName(my_id.getIDURL())+'@'+'somehost.org'
# options['dht_port'] = int(settings.getDHTPort())
# options['udp_port'] = int(settings.getUDPPort())
# udp.listen(int(settings.getUDPPort()))
# dht_service.init(settings.getDHTPort())
# dht_service.connect()
# udp_node.A('go-online', options)
reactor.addSystemEventTrigger('before', 'shutdown', gateway.shutdown)
gateway.init()
gateway.start()
def _ok_to_send(transport, oldstate, newstate):
if newstate != 'LISTENING':
return
# [filename] [peer idurl]
if len(sys.argv) >= 3:
# bpio.WriteFile(sys.argv[1]+'.signed', p.Serialize())
def _try_reconnect():
sess = udp_session.get_by_peer_id(sys.argv[2])
reconnect = False
if not sess:
reconnect = True
print('sessions',
list(udp_session.sessions_by_peer_id().keys()))
print([
s.peer_id
for s in list(udp_session.sessions().values())
])
else:
if sess.state != 'CONNECTED':
print('state: ', sess.state)
reconnect = True
if reconnect:
print('reconnect', sess)
udp_session.add_pending_outbox_file(
sys.argv[1] + '.signed', sys.argv[2], 'descr',
Deferred(), False)
udp_node.A('connect', sys.argv[2])
reactor.callLater(0.5, _try_reconnect)
def _try_connect():
if udp_node.A().state == 'LISTEN':
print('connect')
gateway.stop_packets_timeout_loop()
udp_session.add_pending_outbox_file(
sys.argv[1] + '.signed', sys.argv[2], 'descr',
Deferred(), False)
udp_node.A('connect', sys.argv[2])
reactor.callLater(5, _try_reconnect)
else:
reactor.callLater(1, _try_connect)
# _try_connect()
def _send(c):
from transport.udp import udp_stream
for idurl in sys.argv[2:]:
print('_send', list(udp_stream.streams().keys()))
p = signed.Packet(commands.Data(), my_id.getIDURL(),
my_id.getIDURL(), 'packet%d' % c,
bpio.ReadBinaryFile(sys.argv[1]), idurl)
gateway.outbox(p)
if c > 1:
reactor.callLater(0.01, _send, c - 1)
reactor.callLater(0, _send, 15)
gateway.add_transport_state_changed_callback(_ok_to_send)
reactor.run()
def shutdown_tester(self, exit_code):
reactor.addSystemEventTrigger('after', 'shutdown', os._exit, exit_code)
reactor.stop()
