def prompt_user_for_chaincom_details(): """ """ config_file = get_config_file() parser = SafeConfigParser() parser.read(config_file) if not parser.has_section('chain_com'): message = '-' * 15 + '\n' message += 'NOTE: Blockstore currently requires API access to chain.com\n' message += 'for getting unspent outputs. We will add support for using\n' message += 'bitcoind and/or other API providers in the next release.\n' message += '-' * 15 log.info(message) api_key_id = raw_input("Enter chain.com API Key ID: ") api_key_secret = raw_input("Enter chain.com API Key Secret: ") if api_key_id != '' and api_key_secret != '': parser.add_section('chain_com') parser.set('chain_com', 'api_key_id', api_key_id) parser.set('chain_com', 'api_key_secret', api_key_secret) fout = open(config_file, 'w') parser.write(fout) # update in config as well (which was already initialized) config.CHAIN_COM_API_ID = api_key_id config.CHAIN_COM_API_SECRET = api_key_secret
def init_bitcoind(): """ """ config_file = get_config_file() parser = SafeConfigParser() parser.read(config_file) if parser.has_section('bitcoind'): try: return create_bitcoind_connection() except: return prompt_user_for_bitcoind_details() else: pass else: user_input = raw_input( "Do you have your own bitcoind server? (yes/no): ") if user_input.lower() == "yes" or user_input.lower() == "y": return prompt_user_for_bitcoind_details() else: log.info( "Using default bitcoind server at %s", config.BITCOIND_SERVER) return create_bitcoind_connection()
def process_line_data(self, data, raw_line): if "ack" in data: if data["ack"] == 1: self.process_ack(data["uid_conversation"]) else: log.info("Un-ack received: {message!r}", message=raw_line.rstrip()) if "to" in data: uid_to = data["to"] clients = self.factory.clients.get(uid_to) if clients: uid_conversation = str(uuid.uuid4()) self.factory.waiting_acks[uid_conversation] = { "clients": clients, "channel": self } data = self.serializer.dumps({ 'uid_to': uid_to, 'uid_conversation': uid_conversation }) for client in clients: try: client.sendLine(data) except Exception, e: log.failure("{message!r}", message=e.message) client.transport.loseConnection() callback = reactor.callLater(1.7, PubProtocol.no_ack_timeout, self.factory, self, uid_conversation, uid_to) self.factory.conversation_callbacks[ uid_conversation] = callback else: self.send_uid_no_registered()
def get_application(config): app = Application('Scrapyd') http_port = config.getint('http_port', 6800) bind_address = config.get('bind_address', '0.0.0.0') poll_interval = config.getfloat('poll_interval', 5) poller = QueuePoller(config) eggstorage = FilesystemEggStorage(config) scheduler = SpiderScheduler(config) environment = Environment(config) app.setComponent(IPoller, poller) app.setComponent(IEggStorage, eggstorage) app.setComponent(ISpiderScheduler, scheduler) app.setComponent(IEnvironment, environment) laupath = config.get('launcher', 'scrapyd_mongodb.launcher.Launcher') laucls = load_object(laupath) launcher = laucls(config, app) timer = TimerService(poll_interval, poller.poll) webservice = TCPServer(http_port, server.Site(Root(config, app)), interface=bind_address) log.info( 'Scrapyd web console available at ' 'http://{bind_address}:{http_port}/', bind_address=bind_address, http_port=http_port) launcher.setServiceParent(app) timer.setServiceParent(app) webservice.setServiceParent(app) return app
def handle_data(self, pt, timestamp, sample, marker): if self.Done: if self._cbDone: self._cbDone() return # We need to keep track of whether we were in silence mode or not - # when we go from silent->talking, set the marker bit. Other end # can use this as an excuse to adjust playout buffer. if not self.sending: if not hasattr(self, 'warnedaboutthis'): log.info(("%s.handle_media_sample() should only be called" + " only when it is in sending mode.") % (self, )) if VERBOSE: print "WARNING: warnedaboutthis" self.warnedaboutthis = True return incTS = True #Marker is on first packet after a silent if not self._silent: if marker: marker = 0 self._silent = True incTS = False else: marker = 1 self._silent = False if incTS: #Taking care about ts self.ts += int(timestamp) # Wrapping if self.ts >= TWO_TO_THE_32ND: self.ts = self.ts - TWO_TO_THE_32ND self._send_packet(pt, sample, marker=marker)
def handle_data(self, pt, timestamp, sample, marker): if self.Done: if self._cbDone: self._cbDone() return # We need to keep track of whether we were in silence mode or not - # when we go from silent->talking, set the marker bit. Other end # can use this as an excuse to adjust playout buffer. if not self.sending: if not hasattr(self, 'warnedaboutthis'): log.info(("%s.handle_media_sample() should only be called" + " only when it is in sending mode.") % (self,)) if VERBOSE: print "WARNING: warnedaboutthis" self.warnedaboutthis = True return incTS = True #Marker is on first packet after a silent if not self._silent: if marker: marker = 0 self._silent = True incTS = False else: marker = 1 self._silent = False if incTS: #Taking care about ts self.ts += int(timestamp) # Wrapping if self.ts >= TWO_TO_THE_32ND: self.ts = self.ts - TWO_TO_THE_32ND self._send_packet(pt, sample, marker=marker)
def hs_desc(evt): """ From control-spec: "650" SP "HS_DESC" SP Action SP HSAddress SP AuthType SP HsDir [SP DescriptorID] [SP "REASON=" Reason] [SP "REPLICA=" Replica] """ args = evt.split() subtype = args[0] if subtype == 'UPLOAD': if args[1] == self.hostname[:-6]: attempted_uploads.add(args[3]) elif subtype == 'UPLOADED': # we only need ONE successful upload to happen for the # HS to be reachable. (addr is args[1]) if args[3] in attempted_uploads: confirmed_uploads.add(args[3]) log.info("Uploaded '{}' to '{}'".format(self.hostname, args[3])) # pylint: disable=no-member uploaded.callback(self) elif subtype == 'FAILED': if args[1] == self.hostname[:-6]: failed_uploads.add(args[3]) if failed_uploads == attempted_uploads: msg = "Failed to upload '{}' to: {}".format( self.hostname, ', '.join(failed_uploads), ) uploaded.errback(RuntimeError(msg))
def _sighup_handler(self, signum, frame): self.ufc.configure() # Si hemos cambiado la configuraciĆ³n de base de datos debemos abrir # de nuevo todas las conexiones. log.info("Restarting threadpool...") reactor.getThreadPool().stop() reactor.getThreadPool().start()
def init_bitcoind(): """ """ config_file = get_config_file() parser = SafeConfigParser() parser.read(config_file) if parser.has_section('bitcoind'): try: return create_bitcoind_connection() except: return prompt_user_for_bitcoind_details() else: pass else: user_input = raw_input( "Do you have your own bitcoind server? (yes/no): ") if user_input.lower() == "yes" or user_input.lower() == "y": return prompt_user_for_bitcoind_details() else: log.info("Using default bitcoind server at %s", config.BITCOIND_SERVER) return create_bitcoind_connection()
def runOperation(self, query, *args, **kwargs): """ Run an operation on the master. Note that even though 'operations' (e.g., inserts, deletes, updates) should always be on the master, we still check the query, since it could just be a programming error. """ pool = self.getPoolFor(query) while (pool): try: pool.runOperation(query, *args, **kwargs) break except adbapi.ConnectionLost, e: if (pool == self.master): raise e else: log.info( "Expired slave %s during operation because of %s" % (pool.connkw['host'], str(e))) try: self.slaves.remove(pool) pool.close() except: pass pool = self.getPoolFor(query)
def dispatch(self, msg): match = self.regex.search(msg) if not match: log.debug('Failed to match snort rule-sid in msg: {!r}'.format(msg)) return msg sid = match.group('sid') if self.gid_ignore: try: gid = match.group('gid') except IndexError: pass else: if gid in self.gid_ignore: return msg ts = time() if self.sid_db_ts < ts - self.conf.sid_db_mtime_check_interval: if not os.path.exists(self.conf.paths.sid_db)\ or max(0, *( os.stat(p).st_mtime for p in [self.conf.paths.sid_src, self.conf.paths.refs] if os.path.exists(p) )) > os.stat(self.conf.paths.sid_db).st_mtime: self.update_sid_db() self.sid_db = anydbm.open(self.conf.paths.sid_db) try: ref = force_unicode(self.sid_db[force_bytes(sid)]) except KeyError: log.info('Failed to find refs for sid: {!r} (msg: {!r})'.format(sid, msg)) else: msg += u'\n refs: {}'.format(ref) return msg
def onOpen(self): log.info("WebSocket connection open.") self.users = self.factory.users self.factory.maxuid+=1 self.uid = self.factory.maxuid self.users[self.uid]=self self.sendMessage(json.dumps({"setuid":self.uid}))
def vnclog(): usage = '%prog [options] OUTPUT' description = 'Capture user interactions with a VNC Server' op = optparse.OptionParser(usage=usage, description=description) add_standard_options(op) op.add_option('--listen', metavar='PORT', type='int', help='listen for client connections on PORT [%default]') op.set_defaults(listen=5902) op.add_option('--forever', action='store_true', help='continually accept new connections') op.add_option('--viewer', action='store', metavar='CMD', help='launch an interactive client using CMD [%default]') options, args = op.parse_args() setup_logging(options) options.host, options.port = parse_host(options.server) if len(args) != 1: op.error('incorrect number of arguments') output = args[0] factory = build_proxy(options) if options.forever and os.path.isdir(output): factory.output = output elif options.forever: op.error('--forever requires OUTPUT to be a directory') elif output == '-': factory.output = sys.stdout else: factory.output = open(output, 'w') if options.listen == 0: log.info('accepting connections on ::%d', factory.listen_port) factory.password = options.password if options.viewer: cmdline = '%s localhost::%s' % (options.viewer, factory.listen_port) proc = reactor.spawnProcess(ExitingProcess(), options.viewer, cmdline.split(), env=os.environ) reactor.run() sys.exit(reactor.exit_status)
def request_exit_status(self, data): # exit status is a 32-bit unsigned int in network byte format status = struct.unpack_from(">L", data, 0)[0] log.info("Received exit status request: %d", status) self.exit_status = status self.exit_defer.callback(self) self.running = False return True
def vncdo(): usage = '%prog [options] (CMD CMDARGS|-|filename)' description = 'Command line control of a VNC server' op = VNCDoToolOptionParser(usage=usage, description=description) add_standard_options(op) op.add_option('--delay', action='store', metavar='MILLISECONDS', default=os.environ.get('VNCDOTOOL_DELAY', 0), type='int', help='delay MILLISECONDS between actions [%defaultms]') op.add_option('--force-caps', action='store_true', help='for non-compliant servers, send shift-LETTER, ensures capitalization works') op.add_option('--localcursor', action='store_true', help='mouse pointer drawn client-side, useful when server does not include cursor') op.add_option('--nocursor', action='store_true', help='no mouse pointer in screen captures') op.add_option('-t', '--timeout', action='store', type='int', metavar='TIMEOUT', help='abort if unable to complete all actions within TIMEOUT seconds') op.add_option('-w', '--warp', action='store', type='float', metavar='FACTOR', default=1.0, help='pause time is accelerated by FACTOR [x%default]') options, args = op.parse_args() if not len(args): op.error('no command provided') setup_logging(options) options.host, options.port = parse_host(options.server) log.info('connecting to %s:%s', options.host, options.port) factory = build_tool(options, args) factory.password = options.password if options.localcursor: factory.pseudocusor = True if options.nocursor: factory.nocursor = True if options.force_caps: factory.force_caps = True if options.timeout: message = 'TIMEOUT Exceeded (%ss)' % options.timeout failure = Failure(TimeoutError(message)) reactor.callLater(options.timeout, error, failure) reactor.run() sys.exit(reactor.exit_status)
def receiveUnimplemented( self, seqnum ): """ Called when an unimplemented packet message was received from the device. @param seqnum: SSH message code @type seqnum: integer """ message= "Got 'unimplemented' SSH message, seqnum= %d" % seqnum log.info( message ) transport.SSHClientTransport.receiveUnimplemented(self, seqnum)
def master_disconnected(self, client=None): ServerProtocol.master_disconnected(self, client) if self.master and self.master_reconnect_call is None: if client: message = 'Master connection could not be established' else: message = 'Master connection lost' log.info('%s, reconnecting in 60 seconds...' % message) self.master_reconnect_call = reactor.callLater( 60, self.reconnect_master)
def vncdo(): usage = '%prog [options] (CMD CMDARGS|-|filename)' description = 'Command line control of a VNC server' op = build_optparser(usage, description) op.add_option('--delay', action='store', metavar='MILLISECONDS', default=os.environ.get('VNCDOTOOL_DELAY', 0), type='int', help='delay MILLISECONDS between actions [%defaultms]') op.add_option('--nocursor', action='store_true', help='no mouse pointer in screen captures') op.add_option( '--localcursor', action='store_true', help= 'mouse pointer drawn client-side, useful when server does not include cursor' ) op.add_option('-w', '--warp', action='store', type='float', metavar='FACTOR', default=1.0, help='pause time is accelerated by FACTOR [x%default]') options, args = op.parse_args() if not len(args): op.error('no command provided') setup_logging(options) parse_host(options) log.info('connecting to %s:%s', options.host, options.port) factory = build_tool(options, args) factory.password = options.password if options.nocursor: factory.nocursor = True if options.localcursor: factory.pseudocusor = True reactor.run() sys.exit(reactor.exit_status)
def tryAuth(self, kind): kind = kind.replace('-', '_') if kind != 'publickey': log.info('skipping auth method %s (not supported)' % kind) return log.info('trying to auth with %s!' % kind) f = getattr(self, 'auth_%s' % kind, None) if f: return f() else: return
def run_blockstored(): """ run blockstored """ global bitcoin_opts parser = argparse.ArgumentParser(description="Blockstore Core Daemon version {}".format(config.VERSION)) parser.add_argument("--bitcoind-server", help="the hostname or IP address of the bitcoind RPC server") parser.add_argument("--bitcoind-port", type=int, help="the bitcoind RPC port to connect to") parser.add_argument("--bitcoind-user", help="the username for bitcoind RPC server") parser.add_argument("--bitcoind-passwd", help="the password for bitcoind RPC server") parser.add_argument("--bitcoind-use-https", action="store_true", help="use HTTPS to connect to bitcoind") subparsers = parser.add_subparsers(dest="action", help="the action to be taken") parser_server = subparsers.add_parser("start", help="start the blockstored server") parser_server.add_argument("--foreground", action="store_true", help="start the blockstored server in foreground") parser_server = subparsers.add_parser("stop", help="stop the blockstored server") # Print default help message, if no argument is given if len(sys.argv) == 1: parser.print_help() sys.exit(1) args = parser.parse_args() # propagate options for (argname, config_name) in zip( ["bitcoind_server", "bitcoind_port", "bitcoind_user", "bitcoind_passwd"], ["BITCOIND_SERVER", "BITCOIND_PORT", "BITCOIND_USER", "BITCOIND_PASSWD"], ): if hasattr(args, argname) and getattr(args, argname) is not None: bitcoin_opts[argname] = getattr(args, argname) setattr(config, config_name, getattr(args, argname)) if hasattr(args, "bitcoind_use_https"): if args.bitcoind_use_https: config.BITCOIND_USE_HTTPS = True bitcoin_opts["bitcoind_use_https"] = True if args.action == "start": stop_server() if args.foreground: log.info("Initializing blockstored server in foreground ...") run_server(foreground=True) while 1: stay_alive = True else: log.info("Starting blockstored server ...") run_server() elif args.action == "stop": stop_server()
def tryAuth(self, kind): kind = kind.replace("-", "_") if kind != "publickey": log.info("skipping auth method %s (not supported)" % kind) return log.info("trying to auth with %s!" % kind) f = getattr(self, "auth_%s" % kind, None) if f: return f() else: return
def run_blockmirrord(): """ run blockmirrord """ global blockmirrord global bitcoind global namecoind global cached_namespace signal.signal(signal.SIGINT, signal_handler) bitcoin_opts, parser = blockdaemon.parse_bitcoind_args( return_parser=True ) namecoin_opts, parser = parse_namecoind_args( return_parser=True, parser=parser ) parser.add_argument( "--namespace", help="path to the cached namespace JSON file") subparsers = parser.add_subparsers( dest='action', help='the action to be taken') parser_server = subparsers.add_parser( 'start', help='start the blockmirrord server') parser_server.add_argument( '--foreground', action='store_true', help='start the blockmirrord server in foreground') parser_server = subparsers.add_parser( 'stop', help='stop the blockmirrord server') args, _ = parser.parse_known_args() # did we get a namespace JSON file? if hasattr( args, "namespace" ) and getattr( args, "namespace" ) is not None: namespace_path = args.namespace namespace_json = None log.info("Loading JSON from '%s'" % namespace_path) with open(namespace_path, "r") as namespace_fd: namespace_json = namespace_fd.read() log.info("Parsing JSON") try: cached_namespace = json.loads( namespace_json ) except Exception, e: log.exception(e) exit(1)
def run_server(bitcoind, foreground=False): """ run the blockmirrord server """ global blockmirrord if bitcoind is None: bitcoind = blockdaemon.init_bitcoind(config.BLOCKMIRRORD_WORKING_DIR, config.BLOCKMIRRORD_CONFIG_FILE) from .lib.config import BLOCKMIRRORD_PID_FILE, BLOCKMIRRORD_LOG_FILE from .lib.config import BLOCKMIRRORD_TAC_FILE working_dir = blockdaemon.get_working_dir(config.BLOCKMIRRORD_WORKING_DIR) current_dir = os.path.abspath(os.path.dirname(__file__)) tac_file = os.path.join(current_dir, BLOCKMIRRORD_TAC_FILE) log_file = os.path.join(working_dir, BLOCKMIRRORD_LOG_FILE) pid_file = os.path.join(working_dir, BLOCKMIRRORD_PID_FILE) start_block, current_block = get_index_range() if foreground: command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file) else: command = 'twistd --pidfile=%s --logfile=%s -y %s' % ( pid_file, log_file, tac_file) try: # bring the mirror up to speed refresh_mirror() # begin serving blockmirrord = subprocess.Popen(command, shell=True, preexec_fn=os.setsid) log.info('Blockmirrord successfully started') except IndexError, ie: traceback.print_exc() try: os.killpg(blockmirrord.pid, signal.SIGTERM) except: pass exit(1)
def run_blockstored(): """ run blockstored """ parser = argparse.ArgumentParser( description='Blockstore Core Daemon version {}'.format(config.VERSION)) parser.add_argument( '--bitcoind-server', help='the hostname or IP address of the bitcoind RPC server') parser.add_argument( '--bitcoind-port', type=int, help='the bitcoind RPC port to connect to') parser.add_argument( '--bitcoind-user', help='the username for bitcoind RPC server') parser.add_argument( '--bitcoind-passwd', help='the password for bitcoind RPC server') subparsers = parser.add_subparsers( dest='action', help='the action to be taken') parser_server = subparsers.add_parser( 'start', help='start the blockstored server') parser_server.add_argument( '--foreground', action='store_true', help='start the blockstored server in foreground') parser_server = subparsers.add_parser( 'stop', help='stop the blockstored server') # Print default help message, if no argument is given if len(sys.argv) == 1: parser.print_help() sys.exit(1) args = parser.parse_args() if args.action == 'start': stop_server() if args.foreground: log.info('Initializing blockstored server in foreground ...') run_server(foreground=True) while(1): stay_alive = True else: log.info('Starting blockstored server ...') run_server() elif args.action == 'stop': stop_server()
def render(self, request): if self.config["host_header"] is None: requested_host = request.received_headers['host'].partition(":")[0] else: requested_host = request.requestHeaders.getRawHeaders( self.config["host_header"])[0] log.info(u"new_request|%s|%s|%s" % (requested_host, request.method, request.uri)) request.content.seek(0, 0) server = self.server_pool.get_server(requested_host) self.retry(request, server) return NOT_DONE_YET
def receiveDebug( self, alwaysDisplay, message, lang ): """ Called when a debug message was received from the device. @param alwaysDisplay: boolean-type code to indicate if the message is to be displayed @type alwaysDisplay: integer @param message: debug message from remote device @type message: string @param lang: language code @type lang: integer """ message= "Debug message from remote device (%s): %s" % ( str(lang), str(message) ) log.info( message ) transport.SSHClientTransport.receiveDebug(self, alwaysDisplay, message, lang )
def run_server(foreground=False): """ run the blockstored server """ global bitcoind prompt_user_for_chaincom_details() bitcoind = init_bitcoind() from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE from .lib.config import BLOCKSTORED_TAC_FILE from .lib.config import START_BLOCK working_dir = get_working_dir() current_dir = os.path.abspath(os.path.dirname(__file__)) tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE) log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE) pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE) start_block, current_block = get_index_range() if foreground: command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file) else: command = 'twistd --pidfile=%s --logfile=%s -y %s' % ( pid_file, log_file, tac_file) try: # refresh_index(335563, 335566, initial_index=True) if start_block != current_block: refresh_index(start_block, current_block, initial_index=True) blockstored = subprocess.Popen(command, shell=True, preexec_fn=os.setsid) log.info('Blockstored successfully started') except IndexError, ie: # indicates that we don't have the latest block log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \ "Please verify that your bitcoin provider has " + \ "processed up to block %s.\n" % (START_BLOCK) + \ " Example: bitcoin-cli getblockcount" ) try: os.killpg(blockstored.pid, signal.SIGTERM) except: pass exit(1)
def run_server(foreground=False): """ run the blockstored server """ global bitcoind prompt_user_for_chaincom_details() bitcoind = init_bitcoind() from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE from .lib.config import BLOCKSTORED_TAC_FILE from .lib.config import START_BLOCK working_dir = get_working_dir() current_dir = os.path.abspath(os.path.dirname(__file__)) tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE) log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE) pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE) start_block, current_block = get_index_range() if foreground: command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file) else: command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file, log_file, tac_file) try: # refresh_index(335563, 335566, initial_index=True) if start_block != current_block: refresh_index(start_block, current_block, initial_index=True) blockstored = subprocess.Popen( command, shell=True, preexec_fn=os.setsid) log.info('Blockstored successfully started') except IndexError, ie: # indicates that we don't have the latest block log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \ "Please verify that your bitcoin provider has " + \ "processed up to block %s.\n" % (START_BLOCK) + \ " Example: bitcoin-cli getblockcount" ) try: os.killpg(blockstored.pid, signal.SIGTERM) except: pass exit(1)
def get_server(self, hostname): if self.is_alive(hostname): server = self._get_server(hostname) actual_index = self.alive.index(server) self.alive.pop(actual_index) self.alive.append(server) else: port = find_open_port(exclude=self.ports_in_use) self.ports_in_use.add(port) cmd = self.make_command(hostname, port) plog = os.path.join(self.output_log_dir, hostname + ".log") server = Server(hostname, port, cmd, self.env, plog) log.info(u"start_server|%s" % hostname) self.alive.append(server) self.update() return server
def next(ret, request): if user.authenticated: # succeed to login log.info('A user ' + user.username + ' login from ' + client_ip) # enforce to su to another user #if os.environ['USER']=='root': raise NotImplementedError() self.send_object( request, { 'retcode': 0, 'stdout': self.success_login_metadata(user) }) request.finish() else: login_failed(request)
def data_received(self, peer: Peer, packet: Packet) -> None: ip = peer.address.host current_time = reactor.seconds() try: ServerProtocol.data_received(self, peer, packet) except (NoDataLeft, ValueError): import traceback traceback.print_exc() log.info( 'IP %s was hardbanned for invalid data or possibly DDoS.' % ip) self.hard_bans.add(ip) return dt = reactor.seconds() - current_time if dt > 1.0: log.warn('processing {!r} from {} took {}'.format( packet.data, ip, dt))
def run_server( bitcoind, foreground=False): """ run the blockmirrord server """ global blockmirrord if bitcoind is None: bitcoind = blockdaemon.init_bitcoind( config.BLOCKMIRRORD_WORKING_DIR, config.BLOCKMIRRORD_CONFIG_FILE ) from .lib.config import BLOCKMIRRORD_PID_FILE, BLOCKMIRRORD_LOG_FILE from .lib.config import BLOCKMIRRORD_TAC_FILE working_dir = blockdaemon.get_working_dir( config.BLOCKMIRRORD_WORKING_DIR ) current_dir = os.path.abspath(os.path.dirname(__file__)) tac_file = os.path.join(current_dir, BLOCKMIRRORD_TAC_FILE) log_file = os.path.join(working_dir, BLOCKMIRRORD_LOG_FILE) pid_file = os.path.join(working_dir, BLOCKMIRRORD_PID_FILE) start_block, current_block = get_index_range() if foreground: command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file) else: command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file, log_file, tac_file) try: # bring the mirror up to speed refresh_mirror() # begin serving blockmirrord = subprocess.Popen( command, shell=True, preexec_fn=os.setsid) log.info('Blockmirrord successfully started') except IndexError, ie: traceback.print_exc() try: os.killpg(blockmirrord.pid, signal.SIGTERM) except: pass exit(1)
def run_server(foreground=False): """ run the blockstored server """ global bitcoind prompt_user_for_chaincom_details() bitcoind = init_bitcoind() from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE from .lib.config import BLOCKSTORED_TAC_FILE from .lib.config import START_BLOCK working_dir = get_working_dir() current_dir = os.path.abspath(os.path.dirname(__file__)) tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE) log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE) pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE) start_block, current_block = get_index_range() if foreground: command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file) else: command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file, log_file, tac_file) try: # refresh_index(335563, 335566, initial_index=True) if start_block != current_block: refresh_index(start_block, current_block, initial_index=True) blockstored = subprocess.Popen( command, shell=True, preexec_fn=os.setsid) log.info('Blockstored successfully started') except Exception as e: log.debug(e) log.info('Exiting blockstored server') try: os.killpg(blockstored.pid, signal.SIGTERM) except: pass exit(1)
def ready_client(reactor, netloc, topic): """ Connect to a Kafka broker and wait for the named topic to exist. This assumes that ``auto.create.topics.enable`` is set in the broker configuration. :raises: `KafkaUnavailableError` if unable to connect. """ client = KafkaClient(netloc, reactor=reactor) e = True while e: yield client.load_metadata_for_topics(topic) e = client.metadata_error_for_topic(topic) if e: log.info("Error getting metadata for topic %r: %s (will retry)", topic, e) defer.returnValue(client)
def get_index_range(start_block=0): """ """ from lib.config import FIRST_BLOCK_MAINNET if start_block == 0: start_block = FIRST_BLOCK_MAINNET try: current_block = int(bitcoind.getblockcount()) except: log.info("ERROR: Cannot connect to bitcoind") user_input = raw_input( "Do you want to re-enter bitcoind server configs? (yes/no): ") if user_input.lower() == "yes" or user_input.lower() == "y": prompt_user_for_bitcoind_details() log.info("Exiting. Restart blockstored to try the new configs.") exit(1) else: exit(1) working_dir = get_working_dir() lastblock_file = os.path.join( working_dir, config.BLOCKSTORED_LASTBLOCK_FILE) saved_block = 0 if os.path.isfile(lastblock_file): fin = open(lastblock_file, 'r') saved_block = fin.read() saved_block = int(saved_block) fin.close() if saved_block == 0: pass elif saved_block == current_block: start_block = saved_block elif saved_block < current_block: start_block = saved_block + 1 return start_block, current_block
def vncdo(): usage = '%prog [options] (CMD CMDARGS|-|filename)' description = 'Command line control of a VNC server' op = build_optparser(usage, description) op.add_option('--delay', action='store', metavar='MILLISECONDS', default=os.environ.get('VNCDOTOOL_DELAY', 0), type='int', help='delay MILLISECONDS between actions [%defaultms]') op.add_option('--nocursor', action='store_true', help='no mouse pointer in screen captures') op.add_option('--localcursor', action='store_true', help='mouse pointer drawn client-side, useful when server does not include cursor') op.add_option('-w', '--warp', action='store', type='float', metavar='FACTOR', default=1.0, help='pause time is accelerated by FACTOR [x%default]') options, args = op.parse_args() if not len(args): op.error('no command provided') setup_logging(options) parse_host(options) log.info('connecting to %s:%s', options.host, options.port) factory = build_tool(options, args) factory.password = options.password if options.nocursor: factory.nocursor = True if options.localcursor: factory.pseudocusor = True reactor.run() sys.exit(reactor.exit_status)
def get_index_range(start_block=0): """ """ from lib.config import FIRST_BLOCK_MAINNET if start_block == 0: start_block = FIRST_BLOCK_MAINNET try: current_block = int(bitcoind.getblockcount()) except Exception, e: log.exception(e) log.info("ERROR: Cannot connect to bitcoind") user_input = raw_input("Do you want to re-enter bitcoind server configs? (yes/no): ") if user_input.lower() == "yes" or user_input.lower() == "y": prompt_user_for_bitcoind_details() log.info("Exiting. Restart blockstored to try the new configs.") exit(1) else: exit(1)
def produce(reactor, hosts="localhost:9092"): topic = b"example_topic" stop_time = reactor.seconds() + 60.0 # seconds to run client = yield ready_client(reactor, hosts, topic) producers = [ Producer(client, RoundRobinPartitioner), Producer(client, HashedPartitioner), ] def cb_produce(resp): log.info("Produce got response: %r", resp) def eb_produce(f): log.error("Produce failed", exc_info=(f.type, f.value, f.getTracebackObject())) while reactor.seconds() < stop_time: ds = [ # Wait at least half a second before the next send. task.deferLater(reactor, 0.5, lambda: None), ] # Create some number of random messages and send them to # the producers in parallel. for producer in producers: key, messages = make_messages() d = producer.send_messages(topic, key=key, msgs=messages) d.addCallbacks(cb_produce, eb_produce) ds.append(d) yield defer.gatherResults(ds) log.info("\n") log.info("Time is up, stopping producers...") for p in producers: p.stop() yield client.close()
def runQuery(self, query, *args, **kwargs): """ Run a query on a slave. Note that even though 'queries' (e.g., selects) should always be on a slave, we still check the query, since it could just be a programming error. """ pool = self.getPoolFor(query) while(pool): try: return pool.runQuery(query, *args, **kwargs) except adbapi.ConnectionLost, e: if(pool == self.master): raise e else: log.info("Expired slave %s during query because of %s" % (pool.connkw['host'], str(e))) try: self.slaves.remove(pool) pool.close() except: pass pool = self.getPoolFor(query)
def get_index_range(start_block=0): """ """ from lib.config import FIRST_BLOCK_MAINNET if start_block == 0: start_block = FIRST_BLOCK_MAINNET try: current_block = int(bitcoind.getblockcount()) except Exception, e: log.exception(e) log.info("ERROR: Cannot connect to bitcoind") user_input = raw_input( "Do you want to re-enter bitcoind server configs? (yes/no): ") if user_input.lower() == "yes" or user_input.lower() == "y": prompt_user_for_bitcoind_details() log.info("Exiting. Restart blockstored to try the new configs.") exit(1) else: exit(1)
def produce(reactor, hosts='localhost:9092'): topic = b'example_topic' stop_time = reactor.seconds() + 60.0 # seconds to run client = yield ready_client(reactor, hosts, topic) producers = [ Producer(client, RoundRobinPartitioner), Producer(client, HashedPartitioner), ] def cb_produce(resp): log.info("Produce got response: %r", resp) def eb_produce(f): log.error("Produce failed", exc_info=(f.type, f.value, f.getTracebackObject())) while reactor.seconds() < stop_time: ds = [ # Wait at least half a second before the next send. task.deferLater(reactor, 0.5, lambda: None), ] # Create some number of random messages and send them to # the producers in parallel. for producer in producers: key, messages = make_messages() d = producer.send_messages(topic, key=key, msgs=messages) d.addCallbacks(cb_produce, eb_produce) ds.append(d) yield defer.gatherResults(ds) log.info("\n") log.info("Time is up, stopping producers...") for p in producers: p.stop() yield client.close()
def _deliver(self, message): print "------------------- delivering message ----------------------" print message if not message.has_key("channel"): # print "message has no channel!" log.err("message has no channel!") log.info(str(essage)) return self.lastMessage = message if message.has_key("timestamp"): self.lastTimestamp = message["timestamp"] if message.has_key("id"): self.lastId = message["id"] if len(message["channel"]) > 5 and \ str(message["channel"])[0:5] == "/meta": if not message["successful"]: # print ("error for channel: "+message["channel"]) log.err("error for channel: ", message["channel"]) return if message["channel"] == "/meta/connect": log.msg("connected!") self.connectionId = message["connectionId"] self.connected = True self._processBacklog() elif message["channel"] == "/meta/reconnect": self.connected = True self._processBacklog() elif message["channel"] == "/meta/subscribe": self.subscribed(message["subscription"], message) elif message["channel"] == "/meta/unsubscribe": self.unsubscribed(message["subscription"], message)
def runQuery(self, query, *args, **kwargs): """ Run a query on a slave. Note that even though 'queries' (e.g., selects) should always be on a slave, we still check the query, since it could just be a programming error. """ pool = self.getPoolFor(query) while (pool): try: return pool.runQuery(query, *args, **kwargs) except adbapi.ConnectionLost, e: if (pool == self.master): raise e else: log.info("Expired slave %s during query because of %s" % (pool.connkw['host'], str(e))) try: self.slaves.remove(pool) pool.close() except: pass pool = self.getPoolFor(query)
def main(max_tasks_number): # Restricting the pathes. Trying to access the path, # which is not listed here, would cause 404 error. root = BaseView() root.putChild('add', AddTask()) root.putChild('clear', ClearTasks()) root.putChild('pause', PauseWorkers()) root.putChild('resume', ResumeWorkers()) root.putChild('status', TaskInfo()) root.putChild('stats', LastStats()) root.putChild('maxtasks', UpdateMaxTasks()) # Define the Site with custom logFormatter site = Site(root, logFormatter=timedLogFormatter) site.logRequest = True reactor.listenTCP(SERVICE_PORT, site) log.info("Starting Workers with max-tasks-number=%s" % max_tasks_number) workers_manager.set_max_tasks_number(max_tasks_number) reactor.callLater(0, workers_manager.resume) reactor.run()
def runOperation(self, query, *args, **kwargs): """ Run an operation on the master. Note that even though 'operations' (e.g., inserts, deletes, updates) should always be on the master, we still check the query, since it could just be a programming error. """ pool = self.getPoolFor(query) while(pool): try: pool.runOperation(query, *args, **kwargs) break except adbapi.ConnectionLost, e: if(pool == self.master): raise e else: log.info("Expired slave %s during operation because of %s" % (pool.connkw['host'], str(e))) try: self.slaves.remove(pool) pool.close() except: pass pool = self.getPoolFor(query)
def get_external_ip(self, ip_getter: str) -> Iterator[Deferred]: log.info( 'Retrieving external IP from {!r} to generate server identifier.'. format(ip_getter)) try: ip = yield self.getPage(ip_getter) ip = IPv4Address(ip.strip()) except AddressValueError as e: log.warn('External IP getter service returned invalid data.\n' 'Please check the "ip_getter" setting in your config.') return except Exception as e: log.warn("Getting external IP failed: {reason}", reason=e) return self.ip = ip self.identifier = make_server_identifier(ip, self.port) log.info('Server public ip address: {}:{}'.format(ip, self.port)) log.info('Public aos identifier: {}'.format(self.identifier))
def replaceTwistedLoggers(): """ Visit all Python modules that have been loaded and: - replace L{twisted.python.log} with a L{LegacyLogger} - replace L{twisted.python.log.msg} with a L{LegacyLogger}'s C{msg} - replace L{twisted.python.log.err} with a L{LegacyLogger}'s C{err} """ log = Logger() for moduleName, module in sys.modules.iteritems(): # Oddly, this happens if module is None: continue # Don't patch Twisted's logging module if module in (twisted.python, twisted.python.log): continue # Don't patch this module if moduleName is __name__: continue for name, obj in module.__dict__.iteritems(): newLogger = Logger(namespace=module.__name__) legacyLogger = LegacyLogger(logger=newLogger) if obj is twisted.python.log: log.info("Replacing Twisted log module object {0} in {1}" .format(name, module.__name__)) setattr(module, name, legacyLogger) elif obj is twisted.python.log.msg: log.info("Replacing Twisted log.msg object {0} in {1}" .format(name, module.__name__)) setattr(module, name, legacyLogger.msg) elif obj is twisted.python.log.err: log.info("Replacing Twisted log.err object {0} in {1}" .format(name, module.__name__)) setattr(module, name, legacyLogger.err)
def serviceStopped(self): log.info("Service stopped") connection.SSHConnection.serviceStopped(self) if not self.service_stop_defer.called: self.service_stop_defer.callback(self)
def unsubscribeReceived(self, entity): xmppim.PresenceClientProtocol.unsubscribe(self, entity) log.info(u"Received and accepted unsubscribe request from %s" % entity.full())
def log_connected(pcol): log.info('connected to %s' % pcol.name) return pcol