def main(args=None):
from optparse import OptionParser
usage = "usage: %prog [options] command [arguments]\nCommand is one of: " \
"register remove removematching list listmatching ping"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--host", dest="host", help="hostname of the NS")
parser.add_option("-p", "--port", dest="port", type="int",
help="port of the NS (or bc-port if host isn't specified)")
parser.add_option("-u", "--unixsocket", help="Unix domain socket name of the NS")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose output")
options, args = parser.parse_args(args)
if not args or args[0] not in ("register", "remove", "removematching", "list", "listmatching", "ping"):
parser.error("invalid or missing command")
if options.verbose:
print("Locating name server...")
if options.unixsocket:
options.host = "./u:" + options.unixsocket
try:
nameserver = naming.locateNS(options.host, options.port)
except errors.PyroError:
x = sys.exc_info()[1]
print("Failed to locate the name server: %s" % x)
return
if options.verbose:
print("Name server found: %s" % nameserver._pyroUri)
handleCommand(nameserver, options, args)
if options.verbose:
print("Done.")
def get_my_ip():
"""
Try to obtain our external ip (from the pyro nameserver's point of view)
This tries to sidestep the issue of bogus `/etc/hosts` entries and other
local misconfigurations, which often mess up hostname resolution.
If all else fails, fall back to simple `socket.gethostbyname()` lookup.
"""
import socket
try:
from Pyro4 import naming
# we know the nameserver must exist, so use it as our anchor point
ns = naming.locateNS()
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect((ns._pyroUri.host, ns._pyroUri.port))
result, port = s.getsockname()
except:
try:
# see what ifconfig says about our default interface
import commands
result = commands.getoutput("ifconfig").split(
"\n")[1].split()[1][5:]
if len(result.split('.')) != 4:
raise Exception()
except:
# give up, leave the resolution to gethostbyname
result = socket.gethostbyname(socket.gethostname())
return result
def main(args, returnWithoutLooping=False):
from optparse import OptionParser
parser=OptionParser()
parser.add_option("-H","--host", default="localhost", help="hostname to bind server on (default=localhost)")
parser.add_option("-p","--port", type="int", default=0, help="port to bind server on")
parser.add_option("-u","--unixsocket", help="Unix domain socket name to bind server on")
parser.add_option("-n","--naming", action="store_true", default=False, help="register with nameserver")
parser.add_option("-N","--nameserver", action="store_true", default=False, help="also start a nameserver")
parser.add_option("-v","--verbose", action="store_true", default=False, help="verbose output")
parser.add_option("-q","--quiet", action="store_true", default=False, help="don't output anything")
parser.add_option("-k","--key", help="the HMAC key to use")
options,args = parser.parse_args(args)
if options.verbose:
options.quiet=False
if not options.quiet:
print("Starting Pyro's built-in test echo server.")
Pyro4.config.SERVERTYPE="multiplex"
hmac = (options.key or "").encode("utf-8")
Pyro4.config.HMAC_KEY=hmac or Pyro4.config.HMAC_KEY
if not options.quiet and Pyro4.config.HMAC_KEY:
print("HMAC_KEY set to: %s" % Pyro4.config.HMAC_KEY)
nameserver=None
if options.nameserver:
options.naming=True
nameserver=startNameServer(options.host)
d=Pyro4.Daemon(host=options.host, port=options.port, unixsocket=options.unixsocket)
echo=EchoServer()
echo.verbose=options.verbose
objectName="test.echoserver"
uri=d.register(echo, objectName)
if options.naming:
host,port=None,None
if nameserver is not None:
host,port=nameserver.uri.host, nameserver.uri.port
ns=naming.locateNS(host,port)
ns.register(objectName, uri)
if options.verbose:
print("using name server at %s" % ns._pyroUri)
if nameserver is not None:
if nameserver.bc_server:
print("broadcast server running at %s" % nameserver.bc_server.locationStr)
else:
print("not using a broadcast server")
else:
if options.verbose:
print("not using a name server.")
if not options.quiet:
print("object name: %s" % objectName)
print("echo uri: %s" % uri)
print("echoserver running.")
if returnWithoutLooping:
return d,echo,uri # for unit testing
else:
d.requestLoop(loopCondition=lambda:not echo.must_shutdown)
d.close()
def main(args=None, returnWithoutLooping=False):
parser = OptionParser()
parser.add_option("-H", "--host", default="localhost", help="hostname to bind server on (default=%default)")
parser.add_option("-p", "--port", type="int", default=0, help="port to bind server on")
parser.add_option("-u", "--unixsocket", help="Unix domain socket name to bind server on")
parser.add_option("-n", "--naming", action="store_true", default=False, help="register with nameserver")
parser.add_option("-N", "--nameserver", action="store_true", default=False, help="also start a nameserver")
parser.add_option("-v", "--verbose", action="store_true", default=False, help="verbose output")
parser.add_option("-q", "--quiet", action="store_true", default=False, help="don't output anything")
parser.add_option("-k", "--key", help="the HMAC key to use")
options, args = parser.parse_args(args)
if options.verbose:
options.quiet = False
if not options.quiet:
print("Starting Pyro's built-in test echo server.")
Pyro4.config.SERVERTYPE = "multiplex"
hmac = (options.key or "").encode("utf-8")
if not hmac and not options.quiet:
print("Warning: HMAC key not set. Anyone can connect to this server!")
nameserver = None
if options.nameserver:
options.naming = True
nameserver = startNameServer(options.host, hmac=hmac)
d = Pyro4.Daemon(host=options.host, port=options.port, unixsocket=options.unixsocket)
if hmac:
d._pyroHmacKey = hmac
echo = EchoServer()
echo._verbose = options.verbose
objectName = "test.echoserver"
uri = d.register(echo, objectName)
if options.naming:
host, port = None, None
if nameserver is not None:
host, port = nameserver.uri.host, nameserver.uri.port
ns = naming.locateNS(host, port, hmac_key=hmac)
ns.register(objectName, uri)
if options.verbose:
print("using name server at %s" % ns._pyroUri)
if nameserver is not None:
if nameserver.bc_server:
print("broadcast server running at %s" % nameserver.bc_server.locationStr)
else:
print("not using a broadcast server")
else:
if options.verbose:
print("not using a name server.")
if not options.quiet:
print("object name: %s" % objectName)
print("echo uri: %s" % uri)
print("echoserver running.")
if returnWithoutLooping:
return d, echo, uri # for unit testing
else:
d.requestLoop(loopCondition=lambda: not echo._must_shutdown)
d.close()
def main(args):
from optparse import OptionParser
usage = "usage: %prog [options] command [arguments]\nCommand is one of: " \
"register remove removematching list listmatching ping"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--host", dest="host", help="hostname of the NS")
parser.add_option("-p", "--port", dest="port", type="int",
help="port of the NS (or bc-port if host isn't specified)")
parser.add_option("-u", "--unixsocket", help="Unix domain socket name of the NS")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose output")
options, args = parser.parse_args(args)
if not args or args[0] not in ("register", "remove", "removematching", "list", "listmatching", "ping"):
parser.error("invalid or missing command")
if options.verbose:
print("Locating name server...")
if options.unixsocket:
options.host="./u:"+options.unixsocket
try:
nameserver=naming.locateNS(options.host, options.port)
except errors.PyroError:
x = sys.exc_info()[1]
print("Failed to locate the name server: %s" % x)
return
if options.verbose:
print("Name server found: %s" % nameserver._pyroUri)
handleCommand(nameserver, options, args)
if options.verbose:
print("Done.")
def getNodeList(self):
args = ["list"]
usage = "usage: %prog [options] command [arguments]\nCommand is one of: " \
"register remove removematching list listmatching ping"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--host", dest="host", help="hostname of the NS")
parser.add_option("-p", "--port", dest="port", type="int",
help="port of the NS (or bc-port if host isn't specified)")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose output")
options, args = parser.parse_args(args)
Pyro4.config.DOTTEDNAMES = "true"
nameserver=naming.locateNS(options.host, options.port)
if len(args)==1:
resultdict = nameserver.list()
else:
resultdict = nameserver.list(prefix=args[1]), "- prefix '%s'" % args[1]
for name, uri in sorted(resultdict.items()):
Pyro4.config.DOTTEDNAMES = "true"
obj = Pyro4.Proxy("PYRONAME:%s" % (name))
obj.__pyroAttributes = True
#if not isinstance(obj,Pyro4.naming.NameServer):# and not isinstance(obj,ServerNode.ServerNode):
if re.search('\ANode',name):# and not isinstance(obj,ServerNode.ServerNode):
self.nodeList.append(obj)
return self.nodeList
def __init__(self, nameserver_ip, query_count):
self.queue = Queue.Queue()
ns_host = nameserver_ip
self.query_count = query_count
self.ns = naming.locateNS(ns_host)
user_query = input('Search for: ')
for i in range(self.query_count):
self.queue.put(user_query)
self.results = []
def get_nameserver(hmac=None):
global _nameserver
if not _nameserver:
_nameserver = naming.locateNS(hmac_key=hmac)
try:
_nameserver.ping()
return _nameserver
except errors.ConnectionClosedError:
_nameserver = None
print("Connection with nameserver lost, reconnecting...")
return get_nameserver(hmac)
def main(args=None):
from optparse import OptionParser
usage = "usage: %prog [options] command [arguments]\nCommands: " \
"register remove removematching lookup list listmatching\n listmeta_all listmeta_any setmeta ping"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--host", dest="host", help="hostname of the NS")
parser.add_option(
"-p",
"--port",
dest="port",
type="int",
help="port of the NS (or bc-port if host isn't specified)")
parser.add_option("-u",
"--unixsocket",
help="Unix domain socket name of the NS")
parser.add_option("-k", "--key", help="the HMAC key to use (deprecated)")
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
help="verbose output")
options, args = parser.parse_args(args)
if options.key:
warnings.warn(
"using -k to supply HMAC key on the command line is a security problem "
"and is deprecated since Pyro 4.72. See the documentation for an alternative."
)
if "PYRO_HMAC_KEY" in os.environ:
if options.key:
raise SystemExit(
"error: don't use -k and PYRO_HMAC_KEY at the same time")
options.key = os.environ["PYRO_HMAC_KEY"]
if not args or args[0] not in ("register", "remove", "removematching",
"list", "listmatching", "lookup",
"listmeta_all", "listmeta_any", "setmeta",
"ping"):
parser.error("invalid or missing command")
if options.verbose:
print("Locating name server...")
if options.unixsocket:
options.host = "./u:" + options.unixsocket
try:
nameserver = naming.locateNS(options.host,
options.port,
hmac_key=options.key)
except errors.PyroError as x:
print("Error: %s" % x)
return
if options.verbose:
print("Name server found: %s" % nameserver._pyroUri)
handleCommand(nameserver, options, args)
if options.verbose:
print("Done.")
def print_list_result():
try:
name_server = naming.locateNS()
result_dict = name_server.list(return_metadata=True)
except errors.PyroError as x:
print("Error: %s" % x)
return
result_list = []
for name, (uri, metadata) in sorted(result_dict.items()):
# print("%s --> %s" % (name, uri))
print(name)
result_list.append(name)
if metadata:
print(" metadata:", metadata)
return result_list
def main(args=None):
from optparse import OptionParser
usage = "usage: %prog [options] command [arguments]\nCommands: " \
"register remove removematching lookup list listmatching\n listmeta_all listmeta_any setmeta ping"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--host", dest="host", help="hostname of the NS")
parser.add_option(
"-p",
"--port",
dest="port",
type="int",
help="port of the NS (or bc-port if host isn't specified)")
parser.add_option("-u",
"--unixsocket",
help="Unix domain socket name of the NS")
parser.add_option("-k", "--key", help="the HMAC key to use")
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
help="verbose output")
options, args = parser.parse_args(args)
if not args or args[0] not in ("register", "remove", "removematching",
"list", "listmatching", "lookup",
"listmeta_all", "listmeta_any", "setmeta",
"ping"):
parser.error("invalid or missing command")
if options.verbose:
print("Locating name server...")
if options.unixsocket:
options.host = "./u:" + options.unixsocket
try:
nameserver = naming.locateNS(options.host,
options.port,
hmac_key=options.key)
except errors.PyroError as x:
print("Error: %s" % x)
return
if options.verbose:
print("Name server found: %s" % nameserver._pyroUri)
handleCommand(nameserver, options, args)
if options.verbose:
print("Done.")
def main(args=None):
from optparse import OptionParser
usage = "usage: %prog [options] command [arguments]\nCommands: " \
"register remove removematching lookup list listmatching\n listmeta_all listmeta_any setmeta ping"
parser = OptionParser(usage=usage)
parser.add_option("-n", "--host", dest="host", help="hostname of the NS")
parser.add_option("-p", "--port", dest="port", type="int",
help="port of the NS (or bc-port if host isn't specified)")
parser.add_option("-u", "--unixsocket", help="Unix domain socket name of the NS")
parser.add_option("-k", "--key", help="the HMAC key to use (deprecated)")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="verbose output")
options, args = parser.parse_args(args)
if options.key:
warnings.warn("using -k to supply HMAC key on the command line is a security problem "
"and is deprecated since Pyro 4.72. See the documentation for an alternative.")
if "PYRO_HMAC_KEY" in os.environ:
if options.key:
raise SystemExit("error: don't use -k and PYRO_HMAC_KEY at the same time")
options.key = os.environ["PYRO_HMAC_KEY"]
if not args or args[0] not in ("register", "remove", "removematching", "list", "listmatching", "lookup",
"listmeta_all", "listmeta_any", "setmeta", "ping"):
parser.error("invalid or missing command")
if options.verbose:
print("Locating name server...")
if options.unixsocket:
options.host = "./u:" + options.unixsocket
try:
nameserver = naming.locateNS(options.host, options.port, hmac_key=options.key)
except errors.PyroError as x:
print("Error: %s" % x)
return
if options.verbose:
print("Name server found: %s" % nameserver._pyroUri)
handleCommand(nameserver, options, args)
if options.verbose:
print("Done.")
def main(args, returnWithoutLooping=False):
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-H",
"--host",
default="localhost",
help="hostname to bind server on (default=localhost)")
parser.add_option("-p",
"--port",
type="int",
default=0,
help="port to bind server on")
parser.add_option("-u",
"--unixsocket",
help="Unix domain socket name to bind server on")
parser.add_option("-n",
"--naming",
action="store_true",
default=False,
help="register with nameserver")
parser.add_option("-N",
"--nameserver",
action="store_true",
default=False,
help="also start a nameserver")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="verbose output")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="don't output anything")
parser.add_option("-k", "--key", help="the HMAC key to use")
options, args = parser.parse_args(args)
if options.verbose:
options.quiet = False
if not options.quiet:
print("Starting Pyro's built-in test echo server.")
if os.name != "java":
Pyro4.config.SERVERTYPE = "multiplex"
hmac = (options.key or "").encode("utf-8")
Pyro4.config.HMAC_KEY = hmac or Pyro4.config.HMAC_KEY
if not options.quiet and Pyro4.config.HMAC_KEY:
print("HMAC_KEY set to: %s" % Pyro4.config.HMAC_KEY)
nameserver = None
if options.nameserver:
options.naming = True
nameserver = startNameServer(options.host)
d = Pyro4.Daemon(host=options.host,
port=options.port,
unixsocket=options.unixsocket)
echo = EchoServer()
echo.verbose = options.verbose
objectName = "test.echoserver"
uri = d.register(echo, objectName)
if options.naming:
host, port = None, None
if nameserver is not None:
host, port = nameserver.uri.host, nameserver.uri.port
ns = naming.locateNS(host, port)
ns.register(objectName, uri)
if options.verbose:
print("using name server at %s" % ns._pyroUri)
else:
if options.verbose:
print("not using a name server.")
if not options.quiet:
print("object name: %s" % objectName)
print("echo uri: %s" % uri)
print("echoserver running.")
if returnWithoutLooping:
return d, echo, uri # for unit testing
else:
d.requestLoop(loopCondition=lambda: not echo.must_shutdown)
d.close()
def __init__(self):
ns_host = input('enter nameserver ip: ')
self.crawler_count = int(input('enter the number of crawler instances: '))
self.director = Director()
self.ns = naming.locateNS(ns_host)
self.queue = Queue.Queue()
else:
link = 'http://%s' % urlparse(link).netloc
self.datareduce.reduce_links(link)
elif link.startswith('#') or link.startswith('?'):
pass
elif link.startswith('/'):
self.internal_urls.append(urljoin(target, link))
sleep(1)
# if len(self.internal_urls) > 0:
# next_target = self.internal_urls.pop()
# print(next_target)
# self.crawl(next_target)
def return_urls(self):
print(self.datareduce.return_urls())
return self.datareduce.return_urls()
if __name__ == '__main__':
hostname = input('enter host ip: ')
ident = input('enter crawler identifier: ')
crawler = Crawler()
nameserver = naming.locateNS(host='10.7.1.137', port=9090)
print(nameserver)
Pyro4.config.NS_HOST = "10.7.1.137"
Pyro4.Daemon.serveSimple({crawler: 'Crawler%s' % ident},
host=hostname,
ns=True,
verbose=True)
