def prompt_user_for_chaincom_details():
"""
"""
config_file = get_config_file()
parser = SafeConfigParser()
parser.read(config_file)
if not parser.has_section('chain_com'):
message = '-' * 15 + '\n'
message += 'NOTE: Blockstore currently requires API access to chain.com\n'
message += 'for getting unspent outputs. We will add support for using\n'
message += 'bitcoind and/or other API providers in the next release.\n'
message += '-' * 15
log.info(message)
api_key_id = raw_input("Enter chain.com API Key ID: ")
api_key_secret = raw_input("Enter chain.com API Key Secret: ")
if api_key_id != '' and api_key_secret != '':
parser.add_section('chain_com')
parser.set('chain_com', 'api_key_id', api_key_id)
parser.set('chain_com', 'api_key_secret', api_key_secret)
fout = open(config_file, 'w')
parser.write(fout)
# update in config as well (which was already initialized)
config.CHAIN_COM_API_ID = api_key_id
config.CHAIN_COM_API_SECRET = api_key_secret
def init_bitcoind():
"""
"""
config_file = get_config_file()
parser = SafeConfigParser()
parser.read(config_file)
if parser.has_section('bitcoind'):
try:
return create_bitcoind_connection()
except:
return prompt_user_for_bitcoind_details()
else:
pass
else:
user_input = raw_input(
"Do you have your own bitcoind server? (yes/no): ")
if user_input.lower() == "yes" or user_input.lower() == "y":
return prompt_user_for_bitcoind_details()
else:
log.info(
"Using default bitcoind server at %s", config.BITCOIND_SERVER)
return create_bitcoind_connection()
def process_line_data(self, data, raw_line):
if "ack" in data:
if data["ack"] == 1:
self.process_ack(data["uid_conversation"])
else:
log.info("Un-ack received: {message!r}",
message=raw_line.rstrip())
if "to" in data:
uid_to = data["to"]
clients = self.factory.clients.get(uid_to)
if clients:
uid_conversation = str(uuid.uuid4())
self.factory.waiting_acks[uid_conversation] = {
"clients": clients,
"channel": self
}
data = self.serializer.dumps({
'uid_to':
uid_to,
'uid_conversation':
uid_conversation
})
for client in clients:
try:
client.sendLine(data)
except Exception, e:
log.failure("{message!r}", message=e.message)
client.transport.loseConnection()
callback = reactor.callLater(1.7, PubProtocol.no_ack_timeout,
self.factory, self,
uid_conversation, uid_to)
self.factory.conversation_callbacks[
uid_conversation] = callback
else:
self.send_uid_no_registered()
def get_application(config):
app = Application('Scrapyd')
http_port = config.getint('http_port', 6800)
bind_address = config.get('bind_address', '0.0.0.0')
poll_interval = config.getfloat('poll_interval', 5)
poller = QueuePoller(config)
eggstorage = FilesystemEggStorage(config)
scheduler = SpiderScheduler(config)
environment = Environment(config)
app.setComponent(IPoller, poller)
app.setComponent(IEggStorage, eggstorage)
app.setComponent(ISpiderScheduler, scheduler)
app.setComponent(IEnvironment, environment)
laupath = config.get('launcher', 'scrapyd_mongodb.launcher.Launcher')
laucls = load_object(laupath)
launcher = laucls(config, app)
timer = TimerService(poll_interval, poller.poll)
webservice = TCPServer(http_port,
server.Site(Root(config, app)),
interface=bind_address)
log.info(
'Scrapyd web console available at '
'http://{bind_address}:{http_port}/',
bind_address=bind_address,
http_port=http_port)
launcher.setServiceParent(app)
timer.setServiceParent(app)
webservice.setServiceParent(app)
return app
def handle_data(self, pt, timestamp, sample, marker):
if self.Done:
if self._cbDone:
self._cbDone()
return
# We need to keep track of whether we were in silence mode or not -
# when we go from silent->talking, set the marker bit. Other end
# can use this as an excuse to adjust playout buffer.
if not self.sending:
if not hasattr(self, 'warnedaboutthis'):
log.info(("%s.handle_media_sample() should only be called" +
" only when it is in sending mode.") % (self, ))
if VERBOSE:
print "WARNING: warnedaboutthis"
self.warnedaboutthis = True
return
incTS = True
#Marker is on first packet after a silent
if not self._silent:
if marker:
marker = 0
self._silent = True
incTS = False
else:
marker = 1
self._silent = False
if incTS:
#Taking care about ts
self.ts += int(timestamp)
# Wrapping
if self.ts >= TWO_TO_THE_32ND:
self.ts = self.ts - TWO_TO_THE_32ND
self._send_packet(pt, sample, marker=marker)
def handle_data(self, pt, timestamp, sample, marker):
if self.Done:
if self._cbDone:
self._cbDone()
return
# We need to keep track of whether we were in silence mode or not -
# when we go from silent->talking, set the marker bit. Other end
# can use this as an excuse to adjust playout buffer.
if not self.sending:
if not hasattr(self, 'warnedaboutthis'):
log.info(("%s.handle_media_sample() should only be called" +
" only when it is in sending mode.") % (self,))
if VERBOSE:
print "WARNING: warnedaboutthis"
self.warnedaboutthis = True
return
incTS = True
#Marker is on first packet after a silent
if not self._silent:
if marker:
marker = 0
self._silent = True
incTS = False
else:
marker = 1
self._silent = False
if incTS:
#Taking care about ts
self.ts += int(timestamp)
# Wrapping
if self.ts >= TWO_TO_THE_32ND:
self.ts = self.ts - TWO_TO_THE_32ND
self._send_packet(pt, sample, marker=marker)
def hs_desc(evt):
"""
From control-spec:
"650" SP "HS_DESC" SP Action SP HSAddress SP AuthType SP HsDir
[SP DescriptorID] [SP "REASON=" Reason] [SP "REPLICA=" Replica]
"""
args = evt.split()
subtype = args[0]
if subtype == 'UPLOAD':
if args[1] == self.hostname[:-6]:
attempted_uploads.add(args[3])
elif subtype == 'UPLOADED':
# we only need ONE successful upload to happen for the
# HS to be reachable. (addr is args[1])
if args[3] in attempted_uploads:
confirmed_uploads.add(args[3])
log.info("Uploaded '{}' to '{}'".format(self.hostname, args[3])) # pylint: disable=no-member
uploaded.callback(self)
elif subtype == 'FAILED':
if args[1] == self.hostname[:-6]:
failed_uploads.add(args[3])
if failed_uploads == attempted_uploads:
msg = "Failed to upload '{}' to: {}".format(
self.hostname,
', '.join(failed_uploads),
)
uploaded.errback(RuntimeError(msg))
def _sighup_handler(self, signum, frame):
self.ufc.configure()
# Si hemos cambiado la configuración de base de datos debemos abrir
# de nuevo todas las conexiones.
log.info("Restarting threadpool...")
reactor.getThreadPool().stop()
reactor.getThreadPool().start()
def init_bitcoind():
"""
"""
config_file = get_config_file()
parser = SafeConfigParser()
parser.read(config_file)
if parser.has_section('bitcoind'):
try:
return create_bitcoind_connection()
except:
return prompt_user_for_bitcoind_details()
else:
pass
else:
user_input = raw_input(
"Do you have your own bitcoind server? (yes/no): ")
if user_input.lower() == "yes" or user_input.lower() == "y":
return prompt_user_for_bitcoind_details()
else:
log.info("Using default bitcoind server at %s",
config.BITCOIND_SERVER)
return create_bitcoind_connection()
def prompt_user_for_chaincom_details():
"""
"""
config_file = get_config_file()
parser = SafeConfigParser()
parser.read(config_file)
if not parser.has_section('chain_com'):
message = '-' * 15 + '\n'
message += 'NOTE: Blockstore currently requires API access to chain.com\n'
message += 'for getting unspent outputs. We will add support for using\n'
message += 'bitcoind and/or other API providers in the next release.\n'
message += '-' * 15
log.info(message)
api_key_id = raw_input("Enter chain.com API Key ID: ")
api_key_secret = raw_input("Enter chain.com API Key Secret: ")
if api_key_id != '' and api_key_secret != '':
parser.add_section('chain_com')
parser.set('chain_com', 'api_key_id', api_key_id)
parser.set('chain_com', 'api_key_secret', api_key_secret)
fout = open(config_file, 'w')
parser.write(fout)
# update in config as well (which was already initialized)
config.CHAIN_COM_API_ID = api_key_id
config.CHAIN_COM_API_SECRET = api_key_secret
def _sighup_handler(self, signum, frame):
self.ufc.configure()
# Si hemos cambiado la configuración de base de datos debemos abrir
# de nuevo todas las conexiones.
log.info("Restarting threadpool...")
reactor.getThreadPool().stop()
reactor.getThreadPool().start()
def runOperation(self, query, *args, **kwargs):
"""
Run an operation on the master.
Note that even though 'operations' (e.g., inserts, deletes, updates)
should always be on the master, we still check the query, since
it could just be a programming error.
"""
pool = self.getPoolFor(query)
while (pool):
try:
pool.runOperation(query, *args, **kwargs)
break
except adbapi.ConnectionLost, e:
if (pool == self.master):
raise e
else:
log.info(
"Expired slave %s during operation because of %s" %
(pool.connkw['host'], str(e)))
try:
self.slaves.remove(pool)
pool.close()
except:
pass
pool = self.getPoolFor(query)
def dispatch(self, msg):
match = self.regex.search(msg)
if not match:
log.debug('Failed to match snort rule-sid in msg: {!r}'.format(msg))
return msg
sid = match.group('sid')
if self.gid_ignore:
try: gid = match.group('gid')
except IndexError: pass
else:
if gid in self.gid_ignore: return msg
ts = time()
if self.sid_db_ts < ts - self.conf.sid_db_mtime_check_interval:
if not os.path.exists(self.conf.paths.sid_db)\
or max(0, *( os.stat(p).st_mtime
for p in [self.conf.paths.sid_src, self.conf.paths.refs]
if os.path.exists(p) )) > os.stat(self.conf.paths.sid_db).st_mtime:
self.update_sid_db()
self.sid_db = anydbm.open(self.conf.paths.sid_db)
try: ref = force_unicode(self.sid_db[force_bytes(sid)])
except KeyError:
log.info('Failed to find refs for sid: {!r} (msg: {!r})'.format(sid, msg))
else: msg += u'\n refs: {}'.format(ref)
return msg
def onOpen(self):
log.info("WebSocket connection open.")
self.users = self.factory.users
self.factory.maxuid+=1
self.uid = self.factory.maxuid
self.users[self.uid]=self
self.sendMessage(json.dumps({"setuid":self.uid}))
def vnclog():
usage = '%prog [options] OUTPUT'
description = 'Capture user interactions with a VNC Server'
op = optparse.OptionParser(usage=usage, description=description)
add_standard_options(op)
op.add_option('--listen',
metavar='PORT',
type='int',
help='listen for client connections on PORT [%default]')
op.set_defaults(listen=5902)
op.add_option('--forever',
action='store_true',
help='continually accept new connections')
op.add_option('--viewer',
action='store',
metavar='CMD',
help='launch an interactive client using CMD [%default]')
options, args = op.parse_args()
setup_logging(options)
options.host, options.port = parse_host(options.server)
if len(args) != 1:
op.error('incorrect number of arguments')
output = args[0]
factory = build_proxy(options)
if options.forever and os.path.isdir(output):
factory.output = output
elif options.forever:
op.error('--forever requires OUTPUT to be a directory')
elif output == '-':
factory.output = sys.stdout
else:
factory.output = open(output, 'w')
if options.listen == 0:
log.info('accepting connections on ::%d', factory.listen_port)
factory.password = options.password
if options.viewer:
cmdline = '%s localhost::%s' % (options.viewer, factory.listen_port)
proc = reactor.spawnProcess(ExitingProcess(),
options.viewer,
cmdline.split(),
env=os.environ)
reactor.run()
sys.exit(reactor.exit_status)
def request_exit_status(self, data):
# exit status is a 32-bit unsigned int in network byte format
status = struct.unpack_from(">L", data, 0)[0]
log.info("Received exit status request: %d", status)
self.exit_status = status
self.exit_defer.callback(self)
self.running = False
return True
def request_exit_status(self, data):
# exit status is a 32-bit unsigned int in network byte format
status = struct.unpack_from(">L", data, 0)[0]
log.info("Received exit status request: %d", status)
self.exit_status = status
self.exit_defer.callback(self)
self.running = False
return True
def vncdo():
usage = '%prog [options] (CMD CMDARGS|-|filename)'
description = 'Command line control of a VNC server'
op = VNCDoToolOptionParser(usage=usage, description=description)
add_standard_options(op)
op.add_option('--delay', action='store', metavar='MILLISECONDS',
default=os.environ.get('VNCDOTOOL_DELAY', 0), type='int',
help='delay MILLISECONDS between actions [%defaultms]')
op.add_option('--force-caps', action='store_true',
help='for non-compliant servers, send shift-LETTER, ensures capitalization works')
op.add_option('--localcursor', action='store_true',
help='mouse pointer drawn client-side, useful when server does not include cursor')
op.add_option('--nocursor', action='store_true',
help='no mouse pointer in screen captures')
op.add_option('-t', '--timeout', action='store', type='int', metavar='TIMEOUT',
help='abort if unable to complete all actions within TIMEOUT seconds')
op.add_option('-w', '--warp', action='store', type='float',
metavar='FACTOR', default=1.0,
help='pause time is accelerated by FACTOR [x%default]')
options, args = op.parse_args()
if not len(args):
op.error('no command provided')
setup_logging(options)
options.host, options.port = parse_host(options.server)
log.info('connecting to %s:%s', options.host, options.port)
factory = build_tool(options, args)
factory.password = options.password
if options.localcursor:
factory.pseudocusor = True
if options.nocursor:
factory.nocursor = True
if options.force_caps:
factory.force_caps = True
if options.timeout:
message = 'TIMEOUT Exceeded (%ss)' % options.timeout
failure = Failure(TimeoutError(message))
reactor.callLater(options.timeout, error, failure)
reactor.run()
sys.exit(reactor.exit_status)
def receiveUnimplemented( self, seqnum ):
"""
Called when an unimplemented packet message was received from the device.
@param seqnum: SSH message code
@type seqnum: integer
"""
message= "Got 'unimplemented' SSH message, seqnum= %d" % seqnum
log.info( message )
transport.SSHClientTransport.receiveUnimplemented(self, seqnum)
def master_disconnected(self, client=None):
ServerProtocol.master_disconnected(self, client)
if self.master and self.master_reconnect_call is None:
if client:
message = 'Master connection could not be established'
else:
message = 'Master connection lost'
log.info('%s, reconnecting in 60 seconds...' % message)
self.master_reconnect_call = reactor.callLater(
60, self.reconnect_master)
def vncdo():
usage = '%prog [options] (CMD CMDARGS|-|filename)'
description = 'Command line control of a VNC server'
op = build_optparser(usage, description)
op.add_option('--delay',
action='store',
metavar='MILLISECONDS',
default=os.environ.get('VNCDOTOOL_DELAY', 0),
type='int',
help='delay MILLISECONDS between actions [%defaultms]')
op.add_option('--nocursor',
action='store_true',
help='no mouse pointer in screen captures')
op.add_option(
'--localcursor',
action='store_true',
help=
'mouse pointer drawn client-side, useful when server does not include cursor'
)
op.add_option('-w',
'--warp',
action='store',
type='float',
metavar='FACTOR',
default=1.0,
help='pause time is accelerated by FACTOR [x%default]')
options, args = op.parse_args()
if not len(args):
op.error('no command provided')
setup_logging(options)
parse_host(options)
log.info('connecting to %s:%s', options.host, options.port)
factory = build_tool(options, args)
factory.password = options.password
if options.nocursor:
factory.nocursor = True
if options.localcursor:
factory.pseudocusor = True
reactor.run()
sys.exit(reactor.exit_status)
def tryAuth(self, kind):
kind = kind.replace('-', '_')
if kind != 'publickey':
log.info('skipping auth method %s (not supported)' % kind)
return
log.info('trying to auth with %s!' % kind)
f = getattr(self, 'auth_%s' % kind, None)
if f:
return f()
else:
return
def run_blockstored():
""" run blockstored
"""
global bitcoin_opts
parser = argparse.ArgumentParser(description="Blockstore Core Daemon version {}".format(config.VERSION))
parser.add_argument("--bitcoind-server", help="the hostname or IP address of the bitcoind RPC server")
parser.add_argument("--bitcoind-port", type=int, help="the bitcoind RPC port to connect to")
parser.add_argument("--bitcoind-user", help="the username for bitcoind RPC server")
parser.add_argument("--bitcoind-passwd", help="the password for bitcoind RPC server")
parser.add_argument("--bitcoind-use-https", action="store_true", help="use HTTPS to connect to bitcoind")
subparsers = parser.add_subparsers(dest="action", help="the action to be taken")
parser_server = subparsers.add_parser("start", help="start the blockstored server")
parser_server.add_argument("--foreground", action="store_true", help="start the blockstored server in foreground")
parser_server = subparsers.add_parser("stop", help="stop the blockstored server")
# Print default help message, if no argument is given
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# propagate options
for (argname, config_name) in zip(
["bitcoind_server", "bitcoind_port", "bitcoind_user", "bitcoind_passwd"],
["BITCOIND_SERVER", "BITCOIND_PORT", "BITCOIND_USER", "BITCOIND_PASSWD"],
):
if hasattr(args, argname) and getattr(args, argname) is not None:
bitcoin_opts[argname] = getattr(args, argname)
setattr(config, config_name, getattr(args, argname))
if hasattr(args, "bitcoind_use_https"):
if args.bitcoind_use_https:
config.BITCOIND_USE_HTTPS = True
bitcoin_opts["bitcoind_use_https"] = True
if args.action == "start":
stop_server()
if args.foreground:
log.info("Initializing blockstored server in foreground ...")
run_server(foreground=True)
while 1:
stay_alive = True
else:
log.info("Starting blockstored server ...")
run_server()
elif args.action == "stop":
stop_server()
def tryAuth(self, kind):
kind = kind.replace("-", "_")
if kind != "publickey":
log.info("skipping auth method %s (not supported)" % kind)
return
log.info("trying to auth with %s!" % kind)
f = getattr(self, "auth_%s" % kind, None)
if f:
return f()
else:
return
def vnclog():
usage = '%prog [options] OUTPUT'
description = 'Capture user interactions with a VNC Server'
op = optparse.OptionParser(usage=usage, description=description)
add_standard_options(op)
op.add_option('--listen', metavar='PORT', type='int',
help='listen for client connections on PORT [%default]')
op.set_defaults(listen=5902)
op.add_option('--forever', action='store_true',
help='continually accept new connections')
op.add_option('--viewer', action='store', metavar='CMD',
help='launch an interactive client using CMD [%default]')
options, args = op.parse_args()
setup_logging(options)
options.host, options.port = parse_host(options.server)
if len(args) != 1:
op.error('incorrect number of arguments')
output = args[0]
factory = build_proxy(options)
if options.forever and os.path.isdir(output):
factory.output = output
elif options.forever:
op.error('--forever requires OUTPUT to be a directory')
elif output == '-':
factory.output = sys.stdout
else:
factory.output = open(output, 'w')
if options.listen == 0:
log.info('accepting connections on ::%d', factory.listen_port)
factory.password = options.password
if options.viewer:
cmdline = '%s localhost::%s' % (options.viewer, factory.listen_port)
proc = reactor.spawnProcess(ExitingProcess(),
options.viewer, cmdline.split(),
env=os.environ)
reactor.run()
sys.exit(reactor.exit_status)
def run_blockmirrord():
""" run blockmirrord
"""
global blockmirrord
global bitcoind
global namecoind
global cached_namespace
signal.signal(signal.SIGINT, signal_handler)
bitcoin_opts, parser = blockdaemon.parse_bitcoind_args( return_parser=True )
namecoin_opts, parser = parse_namecoind_args( return_parser=True, parser=parser )
parser.add_argument(
"--namespace",
help="path to the cached namespace JSON file")
subparsers = parser.add_subparsers(
dest='action', help='the action to be taken')
parser_server = subparsers.add_parser(
'start',
help='start the blockmirrord server')
parser_server.add_argument(
'--foreground', action='store_true',
help='start the blockmirrord server in foreground')
parser_server = subparsers.add_parser(
'stop',
help='stop the blockmirrord server')
args, _ = parser.parse_known_args()
# did we get a namespace JSON file?
if hasattr( args, "namespace" ) and getattr( args, "namespace" ) is not None:
namespace_path = args.namespace
namespace_json = None
log.info("Loading JSON from '%s'" % namespace_path)
with open(namespace_path, "r") as namespace_fd:
namespace_json = namespace_fd.read()
log.info("Parsing JSON")
try:
cached_namespace = json.loads( namespace_json )
except Exception, e:
log.exception(e)
exit(1)
def run_blockmirrord():
""" run blockmirrord
"""
global blockmirrord
global bitcoind
global namecoind
global cached_namespace
signal.signal(signal.SIGINT, signal_handler)
bitcoin_opts, parser = blockdaemon.parse_bitcoind_args( return_parser=True )
namecoin_opts, parser = parse_namecoind_args( return_parser=True, parser=parser )
parser.add_argument(
"--namespace",
help="path to the cached namespace JSON file")
subparsers = parser.add_subparsers(
dest='action', help='the action to be taken')
parser_server = subparsers.add_parser(
'start',
help='start the blockmirrord server')
parser_server.add_argument(
'--foreground', action='store_true',
help='start the blockmirrord server in foreground')
parser_server = subparsers.add_parser(
'stop',
help='stop the blockmirrord server')
args, _ = parser.parse_known_args()
# did we get a namespace JSON file?
if hasattr( args, "namespace" ) and getattr( args, "namespace" ) is not None:
namespace_path = args.namespace
namespace_json = None
log.info("Loading JSON from '%s'" % namespace_path)
with open(namespace_path, "r") as namespace_fd:
namespace_json = namespace_fd.read()
log.info("Parsing JSON")
try:
cached_namespace = json.loads( namespace_json )
except Exception, e:
log.exception(e)
exit(1)
def run_server(bitcoind, foreground=False):
""" run the blockmirrord server
"""
global blockmirrord
if bitcoind is None:
bitcoind = blockdaemon.init_bitcoind(config.BLOCKMIRRORD_WORKING_DIR,
config.BLOCKMIRRORD_CONFIG_FILE)
from .lib.config import BLOCKMIRRORD_PID_FILE, BLOCKMIRRORD_LOG_FILE
from .lib.config import BLOCKMIRRORD_TAC_FILE
working_dir = blockdaemon.get_working_dir(config.BLOCKMIRRORD_WORKING_DIR)
current_dir = os.path.abspath(os.path.dirname(__file__))
tac_file = os.path.join(current_dir, BLOCKMIRRORD_TAC_FILE)
log_file = os.path.join(working_dir, BLOCKMIRRORD_LOG_FILE)
pid_file = os.path.join(working_dir, BLOCKMIRRORD_PID_FILE)
start_block, current_block = get_index_range()
if foreground:
command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file)
else:
command = 'twistd --pidfile=%s --logfile=%s -y %s' % (
pid_file, log_file, tac_file)
try:
# bring the mirror up to speed
refresh_mirror()
# begin serving
blockmirrord = subprocess.Popen(command,
shell=True,
preexec_fn=os.setsid)
log.info('Blockmirrord successfully started')
except IndexError, ie:
traceback.print_exc()
try:
os.killpg(blockmirrord.pid, signal.SIGTERM)
except:
pass
exit(1)
def run_blockstored():
""" run blockstored
"""
parser = argparse.ArgumentParser(
description='Blockstore Core Daemon version {}'.format(config.VERSION))
parser.add_argument(
'--bitcoind-server',
help='the hostname or IP address of the bitcoind RPC server')
parser.add_argument(
'--bitcoind-port', type=int,
help='the bitcoind RPC port to connect to')
parser.add_argument(
'--bitcoind-user',
help='the username for bitcoind RPC server')
parser.add_argument(
'--bitcoind-passwd',
help='the password for bitcoind RPC server')
subparsers = parser.add_subparsers(
dest='action', help='the action to be taken')
parser_server = subparsers.add_parser(
'start',
help='start the blockstored server')
parser_server.add_argument(
'--foreground', action='store_true',
help='start the blockstored server in foreground')
parser_server = subparsers.add_parser(
'stop',
help='stop the blockstored server')
# Print default help message, if no argument is given
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
if args.action == 'start':
stop_server()
if args.foreground:
log.info('Initializing blockstored server in foreground ...')
run_server(foreground=True)
while(1):
stay_alive = True
else:
log.info('Starting blockstored server ...')
run_server()
elif args.action == 'stop':
stop_server()
def render(self, request):
if self.config["host_header"] is None:
requested_host = request.received_headers['host'].partition(":")[0]
else:
requested_host = request.requestHeaders.getRawHeaders(
self.config["host_header"])[0]
log.info(u"new_request|%s|%s|%s" %
(requested_host, request.method, request.uri))
request.content.seek(0, 0)
server = self.server_pool.get_server(requested_host)
self.retry(request, server)
return NOT_DONE_YET
def receiveDebug( self, alwaysDisplay, message, lang ):
"""
Called when a debug message was received from the device.
@param alwaysDisplay: boolean-type code to indicate if the message is to be displayed
@type alwaysDisplay: integer
@param message: debug message from remote device
@type message: string
@param lang: language code
@type lang: integer
"""
message= "Debug message from remote device (%s): %s" % ( str(lang), str(message) )
log.info( message )
transport.SSHClientTransport.receiveDebug(self, alwaysDisplay, message, lang )
def run_server(foreground=False):
""" run the blockstored server
"""
global bitcoind
prompt_user_for_chaincom_details()
bitcoind = init_bitcoind()
from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE
from .lib.config import BLOCKSTORED_TAC_FILE
from .lib.config import START_BLOCK
working_dir = get_working_dir()
current_dir = os.path.abspath(os.path.dirname(__file__))
tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE)
log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE)
pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE)
start_block, current_block = get_index_range()
if foreground:
command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file)
else:
command = 'twistd --pidfile=%s --logfile=%s -y %s' % (
pid_file, log_file, tac_file)
try:
# refresh_index(335563, 335566, initial_index=True)
if start_block != current_block:
refresh_index(start_block, current_block, initial_index=True)
blockstored = subprocess.Popen(command,
shell=True,
preexec_fn=os.setsid)
log.info('Blockstored successfully started')
except IndexError, ie:
# indicates that we don't have the latest block
log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \
"Please verify that your bitcoin provider has " + \
"processed up to block %s.\n" % (START_BLOCK) + \
" Example: bitcoin-cli getblockcount" )
try:
os.killpg(blockstored.pid, signal.SIGTERM)
except:
pass
exit(1)
def run_server(foreground=False):
""" run the blockstored server
"""
global bitcoind
prompt_user_for_chaincom_details()
bitcoind = init_bitcoind()
from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE
from .lib.config import BLOCKSTORED_TAC_FILE
from .lib.config import START_BLOCK
working_dir = get_working_dir()
current_dir = os.path.abspath(os.path.dirname(__file__))
tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE)
log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE)
pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE)
start_block, current_block = get_index_range()
if foreground:
command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file)
else:
command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file,
log_file,
tac_file)
try:
# refresh_index(335563, 335566, initial_index=True)
if start_block != current_block:
refresh_index(start_block, current_block, initial_index=True)
blockstored = subprocess.Popen(
command, shell=True, preexec_fn=os.setsid)
log.info('Blockstored successfully started')
except IndexError, ie:
# indicates that we don't have the latest block
log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \
"Please verify that your bitcoin provider has " + \
"processed up to block %s.\n" % (START_BLOCK) + \
" Example: bitcoin-cli getblockcount" )
try:
os.killpg(blockstored.pid, signal.SIGTERM)
except:
pass
exit(1)
def get_server(self, hostname):
if self.is_alive(hostname):
server = self._get_server(hostname)
actual_index = self.alive.index(server)
self.alive.pop(actual_index)
self.alive.append(server)
else:
port = find_open_port(exclude=self.ports_in_use)
self.ports_in_use.add(port)
cmd = self.make_command(hostname, port)
plog = os.path.join(self.output_log_dir, hostname + ".log")
server = Server(hostname, port, cmd, self.env, plog)
log.info(u"start_server|%s" % hostname)
self.alive.append(server)
self.update()
return server
def next(ret, request):
if user.authenticated:
# succeed to login
log.info('A user ' + user.username + ' login from ' +
client_ip)
# enforce to su to another user
#if os.environ['USER']=='root': raise NotImplementedError()
self.send_object(
request, {
'retcode': 0,
'stdout': self.success_login_metadata(user)
})
request.finish()
else:
login_failed(request)
def data_received(self, peer: Peer, packet: Packet) -> None:
ip = peer.address.host
current_time = reactor.seconds()
try:
ServerProtocol.data_received(self, peer, packet)
except (NoDataLeft, ValueError):
import traceback
traceback.print_exc()
log.info(
'IP %s was hardbanned for invalid data or possibly DDoS.' % ip)
self.hard_bans.add(ip)
return
dt = reactor.seconds() - current_time
if dt > 1.0:
log.warn('processing {!r} from {} took {}'.format(
packet.data, ip, dt))
def run_server( bitcoind, foreground=False):
""" run the blockmirrord server
"""
global blockmirrord
if bitcoind is None:
bitcoind = blockdaemon.init_bitcoind( config.BLOCKMIRRORD_WORKING_DIR, config.BLOCKMIRRORD_CONFIG_FILE )
from .lib.config import BLOCKMIRRORD_PID_FILE, BLOCKMIRRORD_LOG_FILE
from .lib.config import BLOCKMIRRORD_TAC_FILE
working_dir = blockdaemon.get_working_dir( config.BLOCKMIRRORD_WORKING_DIR )
current_dir = os.path.abspath(os.path.dirname(__file__))
tac_file = os.path.join(current_dir, BLOCKMIRRORD_TAC_FILE)
log_file = os.path.join(working_dir, BLOCKMIRRORD_LOG_FILE)
pid_file = os.path.join(working_dir, BLOCKMIRRORD_PID_FILE)
start_block, current_block = get_index_range()
if foreground:
command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file)
else:
command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file,
log_file,
tac_file)
try:
# bring the mirror up to speed
refresh_mirror()
# begin serving
blockmirrord = subprocess.Popen( command, shell=True, preexec_fn=os.setsid)
log.info('Blockmirrord successfully started')
except IndexError, ie:
traceback.print_exc()
try:
os.killpg(blockmirrord.pid, signal.SIGTERM)
except:
pass
exit(1)
def run_server(foreground=False):
""" run the blockstored server
"""
global bitcoind
prompt_user_for_chaincom_details()
bitcoind = init_bitcoind()
from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE
from .lib.config import BLOCKSTORED_TAC_FILE
from .lib.config import START_BLOCK
working_dir = get_working_dir()
current_dir = os.path.abspath(os.path.dirname(__file__))
tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE)
log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE)
pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE)
start_block, current_block = get_index_range()
if foreground:
command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file)
else:
command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file,
log_file,
tac_file)
try:
# refresh_index(335563, 335566, initial_index=True)
if start_block != current_block:
refresh_index(start_block, current_block, initial_index=True)
blockstored = subprocess.Popen(
command, shell=True, preexec_fn=os.setsid)
log.info('Blockstored successfully started')
except Exception as e:
log.debug(e)
log.info('Exiting blockstored server')
try:
os.killpg(blockstored.pid, signal.SIGTERM)
except:
pass
exit(1)
def ready_client(reactor, netloc, topic):
"""
Connect to a Kafka broker and wait for the named topic to exist.
This assumes that ``auto.create.topics.enable`` is set in the broker
configuration.
:raises: `KafkaUnavailableError` if unable to connect.
"""
client = KafkaClient(netloc, reactor=reactor)
e = True
while e:
yield client.load_metadata_for_topics(topic)
e = client.metadata_error_for_topic(topic)
if e:
log.info("Error getting metadata for topic %r: %s (will retry)",
topic, e)
defer.returnValue(client)
def ready_client(reactor, netloc, topic):
"""
Connect to a Kafka broker and wait for the named topic to exist.
This assumes that ``auto.create.topics.enable`` is set in the broker
configuration.
:raises: `KafkaUnavailableError` if unable to connect.
"""
client = KafkaClient(netloc, reactor=reactor)
e = True
while e:
yield client.load_metadata_for_topics(topic)
e = client.metadata_error_for_topic(topic)
if e:
log.info("Error getting metadata for topic %r: %s (will retry)",
topic, e)
defer.returnValue(client)
def get_index_range(start_block=0):
"""
"""
from lib.config import FIRST_BLOCK_MAINNET
if start_block == 0:
start_block = FIRST_BLOCK_MAINNET
try:
current_block = int(bitcoind.getblockcount())
except:
log.info("ERROR: Cannot connect to bitcoind")
user_input = raw_input(
"Do you want to re-enter bitcoind server configs? (yes/no): ")
if user_input.lower() == "yes" or user_input.lower() == "y":
prompt_user_for_bitcoind_details()
log.info("Exiting. Restart blockstored to try the new configs.")
exit(1)
else:
exit(1)
working_dir = get_working_dir()
lastblock_file = os.path.join(
working_dir, config.BLOCKSTORED_LASTBLOCK_FILE)
saved_block = 0
if os.path.isfile(lastblock_file):
fin = open(lastblock_file, 'r')
saved_block = fin.read()
saved_block = int(saved_block)
fin.close()
if saved_block == 0:
pass
elif saved_block == current_block:
start_block = saved_block
elif saved_block < current_block:
start_block = saved_block + 1
return start_block, current_block
def vncdo():
usage = '%prog [options] (CMD CMDARGS|-|filename)'
description = 'Command line control of a VNC server'
op = build_optparser(usage, description)
op.add_option('--delay', action='store', metavar='MILLISECONDS',
default=os.environ.get('VNCDOTOOL_DELAY', 0), type='int',
help='delay MILLISECONDS between actions [%defaultms]')
op.add_option('--nocursor', action='store_true',
help='no mouse pointer in screen captures')
op.add_option('--localcursor', action='store_true',
help='mouse pointer drawn client-side, useful when server does not include cursor')
op.add_option('-w', '--warp', action='store', type='float',
metavar='FACTOR', default=1.0,
help='pause time is accelerated by FACTOR [x%default]')
options, args = op.parse_args()
if not len(args):
op.error('no command provided')
setup_logging(options)
parse_host(options)
log.info('connecting to %s:%s', options.host, options.port)
factory = build_tool(options, args)
factory.password = options.password
if options.nocursor:
factory.nocursor = True
if options.localcursor:
factory.pseudocusor = True
reactor.run()
sys.exit(reactor.exit_status)
def get_index_range(start_block=0):
"""
"""
from lib.config import FIRST_BLOCK_MAINNET
if start_block == 0:
start_block = FIRST_BLOCK_MAINNET
try:
current_block = int(bitcoind.getblockcount())
except Exception, e:
log.exception(e)
log.info("ERROR: Cannot connect to bitcoind")
user_input = raw_input("Do you want to re-enter bitcoind server configs? (yes/no): ")
if user_input.lower() == "yes" or user_input.lower() == "y":
prompt_user_for_bitcoind_details()
log.info("Exiting. Restart blockstored to try the new configs.")
exit(1)
else:
exit(1)
def produce(reactor, hosts="localhost:9092"):
topic = b"example_topic"
stop_time = reactor.seconds() + 60.0 # seconds to run
client = yield ready_client(reactor, hosts, topic)
producers = [
Producer(client, RoundRobinPartitioner),
Producer(client, HashedPartitioner),
]
def cb_produce(resp):
log.info("Produce got response: %r", resp)
def eb_produce(f):
log.error("Produce failed",
exc_info=(f.type, f.value, f.getTracebackObject()))
while reactor.seconds() < stop_time:
ds = [
# Wait at least half a second before the next send.
task.deferLater(reactor, 0.5, lambda: None),
]
# Create some number of random messages and send them to
# the producers in parallel.
for producer in producers:
key, messages = make_messages()
d = producer.send_messages(topic, key=key, msgs=messages)
d.addCallbacks(cb_produce, eb_produce)
ds.append(d)
yield defer.gatherResults(ds)
log.info("\n")
log.info("Time is up, stopping producers...")
for p in producers:
p.stop()
yield client.close()
def runQuery(self, query, *args, **kwargs):
"""
Run a query on a slave.
Note that even though 'queries' (e.g., selects) should always be on
a slave, we still check the query, since it could just be a programming error.
"""
pool = self.getPoolFor(query)
while(pool):
try:
return pool.runQuery(query, *args, **kwargs)
except adbapi.ConnectionLost, e:
if(pool == self.master):
raise e
else:
log.info("Expired slave %s during query because of %s" % (pool.connkw['host'], str(e)))
try:
self.slaves.remove(pool)
pool.close()
except:
pass
pool = self.getPoolFor(query)
def get_index_range(start_block=0):
"""
"""
from lib.config import FIRST_BLOCK_MAINNET
if start_block == 0:
start_block = FIRST_BLOCK_MAINNET
try:
current_block = int(bitcoind.getblockcount())
except Exception, e:
log.exception(e)
log.info("ERROR: Cannot connect to bitcoind")
user_input = raw_input(
"Do you want to re-enter bitcoind server configs? (yes/no): ")
if user_input.lower() == "yes" or user_input.lower() == "y":
prompt_user_for_bitcoind_details()
log.info("Exiting. Restart blockstored to try the new configs.")
exit(1)
else:
exit(1)
def produce(reactor, hosts='localhost:9092'):
topic = b'example_topic'
stop_time = reactor.seconds() + 60.0 # seconds to run
client = yield ready_client(reactor, hosts, topic)
producers = [
Producer(client, RoundRobinPartitioner),
Producer(client, HashedPartitioner),
]
def cb_produce(resp):
log.info("Produce got response: %r", resp)
def eb_produce(f):
log.error("Produce failed", exc_info=(f.type, f.value, f.getTracebackObject()))
while reactor.seconds() < stop_time:
ds = [
# Wait at least half a second before the next send.
task.deferLater(reactor, 0.5, lambda: None),
]
# Create some number of random messages and send them to
# the producers in parallel.
for producer in producers:
key, messages = make_messages()
d = producer.send_messages(topic, key=key, msgs=messages)
d.addCallbacks(cb_produce, eb_produce)
ds.append(d)
yield defer.gatherResults(ds)
log.info("\n")
log.info("Time is up, stopping producers...")
for p in producers:
p.stop()
yield client.close()
def _deliver(self, message):
print "------------------- delivering message ----------------------"
print message
if not message.has_key("channel"):
# print "message has no channel!"
log.err("message has no channel!")
log.info(str(essage))
return
self.lastMessage = message
if message.has_key("timestamp"):
self.lastTimestamp = message["timestamp"]
if message.has_key("id"):
self.lastId = message["id"]
if len(message["channel"]) > 5 and \
str(message["channel"])[0:5] == "/meta":
if not message["successful"]:
# print ("error for channel: "+message["channel"])
log.err("error for channel: ", message["channel"])
return
if message["channel"] == "/meta/connect":
log.msg("connected!")
self.connectionId = message["connectionId"]
self.connected = True
self._processBacklog()
elif message["channel"] == "/meta/reconnect":
self.connected = True
self._processBacklog()
elif message["channel"] == "/meta/subscribe":
self.subscribed(message["subscription"], message)
elif message["channel"] == "/meta/unsubscribe":
self.unsubscribed(message["subscription"], message)
def runQuery(self, query, *args, **kwargs):
"""
Run a query on a slave.
Note that even though 'queries' (e.g., selects) should always be on
a slave, we still check the query, since it could just be a programming error.
"""
pool = self.getPoolFor(query)
while (pool):
try:
return pool.runQuery(query, *args, **kwargs)
except adbapi.ConnectionLost, e:
if (pool == self.master):
raise e
else:
log.info("Expired slave %s during query because of %s" %
(pool.connkw['host'], str(e)))
try:
self.slaves.remove(pool)
pool.close()
except:
pass
pool = self.getPoolFor(query)
def main(max_tasks_number):
# Restricting the pathes. Trying to access the path,
# which is not listed here, would cause 404 error.
root = BaseView()
root.putChild('add', AddTask())
root.putChild('clear', ClearTasks())
root.putChild('pause', PauseWorkers())
root.putChild('resume', ResumeWorkers())
root.putChild('status', TaskInfo())
root.putChild('stats', LastStats())
root.putChild('maxtasks', UpdateMaxTasks())
# Define the Site with custom logFormatter
site = Site(root, logFormatter=timedLogFormatter)
site.logRequest = True
reactor.listenTCP(SERVICE_PORT, site)
log.info("Starting Workers with max-tasks-number=%s" % max_tasks_number)
workers_manager.set_max_tasks_number(max_tasks_number)
reactor.callLater(0, workers_manager.resume)
reactor.run()
def runOperation(self, query, *args, **kwargs):
"""
Run an operation on the master.
Note that even though 'operations' (e.g., inserts, deletes, updates)
should always be on the master, we still check the query, since
it could just be a programming error.
"""
pool = self.getPoolFor(query)
while(pool):
try:
pool.runOperation(query, *args, **kwargs)
break
except adbapi.ConnectionLost, e:
if(pool == self.master):
raise e
else:
log.info("Expired slave %s during operation because of %s" % (pool.connkw['host'], str(e)))
try:
self.slaves.remove(pool)
pool.close()
except:
pass
pool = self.getPoolFor(query)
def get_external_ip(self, ip_getter: str) -> Iterator[Deferred]:
log.info(
'Retrieving external IP from {!r} to generate server identifier.'.
format(ip_getter))
try:
ip = yield self.getPage(ip_getter)
ip = IPv4Address(ip.strip())
except AddressValueError as e:
log.warn('External IP getter service returned invalid data.\n'
'Please check the "ip_getter" setting in your config.')
return
except Exception as e:
log.warn("Getting external IP failed: {reason}", reason=e)
return
self.ip = ip
self.identifier = make_server_identifier(ip, self.port)
log.info('Server public ip address: {}:{}'.format(ip, self.port))
log.info('Public aos identifier: {}'.format(self.identifier))
def replaceTwistedLoggers():
"""
Visit all Python modules that have been loaded and:
- replace L{twisted.python.log} with a L{LegacyLogger}
- replace L{twisted.python.log.msg} with a L{LegacyLogger}'s C{msg}
- replace L{twisted.python.log.err} with a L{LegacyLogger}'s C{err}
"""
log = Logger()
for moduleName, module in sys.modules.iteritems():
# Oddly, this happens
if module is None:
continue
# Don't patch Twisted's logging module
if module in (twisted.python, twisted.python.log):
continue
# Don't patch this module
if moduleName is __name__:
continue
for name, obj in module.__dict__.iteritems():
newLogger = Logger(namespace=module.__name__)
legacyLogger = LegacyLogger(logger=newLogger)
if obj is twisted.python.log:
log.info("Replacing Twisted log module object {0} in {1}"
.format(name, module.__name__))
setattr(module, name, legacyLogger)
elif obj is twisted.python.log.msg:
log.info("Replacing Twisted log.msg object {0} in {1}"
.format(name, module.__name__))
setattr(module, name, legacyLogger.msg)
elif obj is twisted.python.log.err:
log.info("Replacing Twisted log.err object {0} in {1}"
.format(name, module.__name__))
setattr(module, name, legacyLogger.err)
def serviceStopped(self):
log.info("Service stopped")
connection.SSHConnection.serviceStopped(self)
if not self.service_stop_defer.called:
self.service_stop_defer.callback(self)
def unsubscribeReceived(self, entity): xmppim.PresenceClientProtocol.unsubscribe(self, entity) log.info(u"Received and accepted unsubscribe request from %s" % entity.full())
def log_connected(pcol):
log.info('connected to %s' % pcol.name)
return pcol
