def optimized_regex_list(self, regex, return_metadata=False):
try:
regex = re.compile(regex + "$") # add end of string marker
except re.error:
x = sys.exc_info()[1]
raise NamingError("invalid regex: " + str(x))
with self.lock:
try:
with closing(dbm.open(self.dbmfile)) as db:
result = {}
if hasattr(db, "items"):
for key, value in db.items():
key = key.decode("utf-8")
if regex.match(key):
uri = value.decode("utf-8")
result[key] = (
uri, frozenset()
) if return_metadata else uri # always return empty metadata
else:
for key in db.keys():
keystr = key.decode("utf-8")
if regex.match(keystr):
uri = db[key].decode("utf-8")
result[keystr] = (
uri, frozenset()
) if return_metadata else uri # always return empty metadata
return result
except dbm.error as e:
raise NamingError("dbm error in optimized_regex_list: " +
str(e))
def __setitem__(self, key, value):
uri, metadata = value
try:
with closing(sqlite3.connect(self.dbfile)) as db:
cursor = db.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
dbid = cursor.execute("SELECT id FROM pyro_names WHERE name=?",
(key, )).fetchone()
if dbid:
dbid = dbid[0]
cursor.execute("DELETE FROM pyro_metadata WHERE object=?",
(dbid, ))
cursor.execute("DELETE FROM pyro_names WHERE id=?",
(dbid, ))
cursor.execute("INSERT INTO pyro_names(name, uri) VALUES(?,?)",
(key, uri))
if metadata:
object_id = cursor.lastrowid
for m in metadata:
cursor.execute(
"INSERT INTO pyro_metadata(object, metadata) VALUES (?,?)",
(object_id, m))
cursor.close()
db.commit()
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in setitem: " + str(e))
def optimized_metadata_search(self,
metadata_all=None,
metadata_any=None,
return_metadata=False):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
if metadata_any:
# any of the given metadata
params = list(metadata_any)
sql = "SELECT id, name, uri FROM pyro_names WHERE id IN (SELECT object FROM pyro_metadata WHERE metadata IN ({seq}))" \
.format(seq=",".join(['?']*len(metadata_any)))
else:
# all of the given metadata
params = list(metadata_all)
params.append(len(metadata_all))
sql = "SELECT id, name, uri FROM pyro_names WHERE id IN (SELECT object FROM pyro_metadata WHERE metadata IN ({seq}) " \
"GROUP BY object HAVING COUNT(metadata)=?)".format(seq=",".join(['?']*len(metadata_all)))
result = db.execute(sql, params).fetchall()
if return_metadata:
names = {}
for dbid, name, uri in result:
metadata = {
m[0]
for m in db.execute(
"SELECT metadata FROM pyro_metadata WHERE object=?",
(dbid, )).fetchall()
}
names[name] = uri, metadata
else:
names = {name: uri for (dbid, name, uri) in result}
return names
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in optimized_metadata_search: " +
str(e))
def _open(self):
logger.debug("trying Pyro nameserver to find the broker daemon")
conninfo = self.client
try:
nameserver = pyro.locateNS(host=conninfo.hostname,
port=self.default_port)
except NamingError:
reraise(NamingError, NamingError(E_NAMESERVER.format(conninfo)),
sys.exc_info()[2])
try:
# name of registered pyro object
uri = nameserver.lookup(conninfo.virtual_host)
return pyro.Proxy(uri)
except NamingError:
reraise(NamingError, NamingError(E_LOOKUP.format(conninfo)),
sys.exc_info()[2])
def resolve(uri, hmac_key=None):
"""
Resolve a 'magic' uri (PYRONAME, PYROMETA) into the direct PYRO uri.
It finds a name server, and use that to resolve a PYRONAME uri into the direct PYRO uri pointing to the named object.
If uri is already a PYRO uri, it is returned unmodified.
You can consider this a shortcut function so that you don't have to locate and use a name server proxy yourself.
Note: if you need to resolve more than a few names, consider using the name server directly instead of repeatedly
calling this function, to avoid the name server lookup overhead from each call.
"""
if isinstance(uri, basestring):
uri = core.URI(uri)
elif not isinstance(uri, core.URI):
raise TypeError("can only resolve Pyro URIs")
if uri.protocol == "PYRO":
return uri
log.debug("resolving %s", uri)
if uri.protocol == "PYRONAME":
with locateNS(uri.host, uri.port, hmac_key=hmac_key) as nameserver:
return nameserver.lookup(uri.object)
elif uri.protocol == "PYROMETA":
with locateNS(uri.host, uri.port, hmac_key=hmac_key) as nameserver:
candidates = nameserver.list(metadata_all=uri.object)
if candidates:
candidate = random.choice(list(candidates.values()))
log.debug("resolved to candidate %s", candidate)
return core.URI(candidate)
raise NamingError(
"no registrations available with desired metadata properties %s"
% uri.object)
else:
raise PyroError("invalid uri protocol")
def optimized_prefix_list(self, prefix, return_metadata=False):
with self.lock:
try:
with closing(dbm.open(self.dbmfile)) as db:
result = {}
if hasattr(db, "items"):
for key, value in db.items():
key = key.decode("utf-8")
if key.startswith(prefix):
uri = value.decode("utf-8")
result[key] = (
uri, frozenset()
) if return_metadata else uri # always return empty metadata
else:
for key in db.keys():
keystr = key.decode("utf-8")
if keystr.startswith(prefix):
uri = db[key].decode("utf-8")
result[keystr] = (
uri, frozenset()
) if return_metadata else uri # always return empty metadata
return result
except dbm.error as e:
raise NamingError("dbm error in optimized_prefix_list: " +
str(e))
def __iter__(self):
with self.lock:
try:
with closing(dbm.open(self.dbmfile)) as db:
return iter([key.decode("utf-8") for key in db.keys()])
except dbm.error as e:
raise NamingError("dbm error in iter: " + str(e))
def __iter__(self):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
result = db.execute("SELECT name FROM pyro_names")
return iter([n[0] for n in result.fetchall()])
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in iter: " + str(e))
def __len__(self):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
return db.execute(
"SELECT count(*) FROM pyro_names").fetchone()[0]
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in len: " + str(e))
def list(self, prefix=None, regex=None):
"""Retrieve the registered items as a dictionary name-to-URI. The URIs
in the resulting dict are strings, not URI objects.
You can filter by prefix or by regex."""
with self.lock:
if prefix:
result = {}
for name in self.namespace:
if name.startswith(prefix):
result[name] = self.namespace[name]
return result
elif regex:
result = {}
try:
regex = re.compile(regex + "$") # add end of string marker
except re.error:
x = sys.exc_info()[1]
raise NamingError("invalid regex: " + str(x))
else:
for name in self.namespace:
if regex.match(name):
result[name] = self.namespace[name]
return result
else:
# just return (a copy of) everything
return self.namespace.copy()
def optimized_metadata_search(self,
metadata_all=None,
metadata_any=None,
return_metadata=False):
if metadata_all or metadata_any:
raise NamingError("DbmStorage doesn't support metadata")
return self.everything(return_metadata)
def __delitem__(self, key):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
db.execute("DELETE FROM pyro_names WHERE name=?", (key, ))
db.commit()
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in delitem: " + str(e))
def __len__(self):
with self.lock:
try:
with closing(dbm.open(self.dbmfile)) as db:
return len(db)
except dbm.error as e:
raise NamingError("dbm error in len: " + str(e))
def __getitem__(self, item):
item = item.encode("utf-8")
with self.lock:
try:
with closing(dbm.open(self.dbmfile)) as db:
return db[item].decode("utf-8")
except dbm.error as e:
raise NamingError("dbm error in getitem: " + str(e))
def __contains__(self, item):
item = item.encode("utf-8")
with self.lock:
try:
with closing(dbm.open(self.dbmfile)) as db:
return item in db
except dbm.error as e:
raise NamingError("dbm error in contains: " + str(e))
def __delitem__(self, key):
key = key.encode("utf-8")
with self.lock:
try:
with closing(dbm.open(self.dbmfile, "w")) as db:
del db[key]
except dbm.error as e:
raise NamingError("dbm error in delitem: " + str(e))
def clear(self):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
db.execute("DELETE FROM pyro_names")
db.execute("VACUUM")
db.commit()
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in clear: " + str(e))
def __contains__(self, item):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
return db.execute(
"SELECT EXISTS(SELECT 1 FROM pyro_names WHERE name=? LIMIT 1)",
(item, )).fetchone()[0]
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in contains: " + str(e))
def remove_items(self, items):
try:
items = [(item, ) for item in items]
with closing(sqlite3.connect(self.dbfile)) as db:
db.executemany("DELETE FROM pyro_names WHERE name=?", items)
db.commit()
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in remove_items: " + str(e))
def __setitem__(self, key, value):
key = key.encode("utf-8")
value = value.encode("utf-8")
with self.lock:
try:
with closing(dbm.open(self.dbmfile, "w")) as db:
db[key] = value
except dbm.error as e:
raise NamingError("dbm error in setitem: " + str(e))
def __setitem__(self, key, value):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
db.execute("DELETE FROM pyro_names WHERE name=?", (key, ))
db.execute("INSERT INTO pyro_names(name, uri) VALUES(?,?)",
(key, value))
db.commit()
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in setitem: " + str(e))
def everything(self):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
result = db.execute("SELECT name, uri FROM pyro_names")
names = {}
for name, uri in result.fetchall():
names[name] = uri
return names
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in everything: " + str(e))
def __getitem__(self, item):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
result = db.execute("SELECT uri FROM pyro_names WHERE name=?",
(item, )).fetchone()
if result:
return result[0]
else:
raise KeyError(item)
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in getitem: " + str(e))
def remove_items(self, items):
with self.lock:
try:
with closing(dbm.open(self.dbmfile, "w")) as db:
for item in items:
try:
del db[item.encode("utf-8")]
except KeyError:
pass
except dbm.error as e:
raise NamingError("dbm error in remove_items: " + str(e))
def clear(self):
with self.lock:
try:
with closing(dbm.open(self.dbmfile, "w")) as db:
if hasattr(db, "clear"):
db.clear()
else:
for key in db.keys():
del db[key]
except dbm.error as e:
raise NamingError("dbm error in clear: " + str(e))
def _open(self):
conninfo = self.client
pyro.config.HMAC_KEY = conninfo.virtual_host
try:
nameserver = pyro.locateNS(host=conninfo.hostname,
port=self.default_port)
# name of registered pyro object
uri = nameserver.lookup(conninfo.virtual_host)
return pyro.Proxy(uri)
except NamingError:
reraise(NamingError, NamingError(E_LOOKUP.format(conninfo)),
sys.exc_info()[2])
def clear(self):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
db.execute("PRAGMA foreign_keys=ON")
db.execute("DELETE FROM pyro_metadata")
db.execute("DELETE FROM pyro_names")
db.commit()
with closing(sqlite3.connect(self.dbfile,
isolation_level=None)) as db:
db.execute("VACUUM") # this cannot run inside a transaction.
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in clear: " + str(e))
def __setitem__(self, key, value):
uri, metadata = value
if metadata:
log.warning(
"DbmStorage doesn't support metadata, silently discarded")
key = key.encode("utf-8")
uri = uri.encode("utf-8")
with self.lock:
try:
with closing(dbm.open(self.dbmfile, "w")) as db:
db[key] = uri
except dbm.error as e:
raise NamingError("dbm error in setitem: " + str(e))
def optimized_prefix_list(self, prefix):
try:
with closing(sqlite3.connect(self.dbfile)) as db:
result = db.execute(
"SELECT name, uri FROM pyro_names WHERE name LIKE ?",
(prefix + '%', ))
names = {}
for name, uri in result.fetchall():
names[name] = uri
return names
except sqlite3.DatabaseError as e:
raise NamingError("sqlite error in optimized_prefix_list: " +
str(e))
def lookup(self, name, return_metadata=False):
"""
Lookup the given name, returns an URI if found.
Returns tuple (uri, metadata) if return_metadata is True.
"""
try:
uri, metadata = self.storage[name]
uri = core.URI(uri)
if return_metadata:
metadata = list(metadata) if metadata else []
return uri, metadata
return uri
except KeyError:
raise NamingError("unknown name: " + name)
def locateNS(host=None, port=None, broadcast=True, hmac_key=None):
"""Get a proxy for a name server somewhere in the network."""
if host is None:
# first try localhost if we have a good chance of finding it there
if Pyro4.config.NS_HOST in ("localhost", "::1") or Pyro4.config.NS_HOST.startswith("127."):
host = Pyro4.config.NS_HOST
if ":" in host: # ipv6
host = "[%s]" % host
uristring = "PYRO:%s@%s:%d" % (Pyro4.constants.NAMESERVER_NAME, host, port or Pyro4.config.NS_PORT)
log.debug("locating the NS: %s", uristring)
proxy = core.Proxy(uristring)
proxy._pyroHmacKey = hmac_key
try:
proxy.ping()
log.debug("located NS")
return proxy
except PyroError:
pass
if broadcast:
# broadcast lookup
if not port:
port = Pyro4.config.NS_BCPORT
log.debug("broadcast locate")
sock = Pyro4.socketutil.createBroadcastSocket(reuseaddr=Pyro4.config.SOCK_REUSE, timeout=0.7)
for _ in range(3):
try:
for bcaddr in Pyro4.config.parseAddressesString(Pyro4.config.BROADCAST_ADDRS):
try:
sock.sendto(BroadcastServer.REQUEST_NSURI, 0, (bcaddr, port))
except socket.error:
x = sys.exc_info()[1]
err = getattr(x, "errno", x.args[0])
# handle some errno's that some platforms like to throw:
if err not in Pyro4.socketutil.ERRNO_EADDRNOTAVAIL and err not in Pyro4.socketutil.ERRNO_EADDRINUSE:
raise
data, _ = sock.recvfrom(100)
sock.close()
if sys.version_info >= (3, 0):
data = data.decode("iso-8859-1")
log.debug("located NS: %s", data)
proxy = core.Proxy(data)
proxy._pyroHmacKey = hmac_key
return proxy
except socket.timeout:
continue
try:
sock.shutdown(socket.SHUT_RDWR)
except (OSError, socket.error):
pass
sock.close()
log.debug("broadcast locate failed, try direct connection on NS_HOST")
else:
log.debug("skipping broadcast lookup")
# broadcast failed or skipped, try PYRO directly on specific host
host = Pyro4.config.NS_HOST
port = Pyro4.config.NS_PORT
# pyro direct lookup
if not port:
port = Pyro4.config.NS_PORT
if ":" in host:
host = "[%s]" % host
if core.URI.isUnixsockLocation(host):
uristring = "PYRO:%s@%s" % (Pyro4.constants.NAMESERVER_NAME, host)
else:
uristring = "PYRO:%s@%s:%d" % (Pyro4.constants.NAMESERVER_NAME, host, port)
uri = core.URI(uristring)
log.debug("locating the NS: %s", uri)
proxy = core.Proxy(uri)
proxy._pyroHmacKey = hmac_key
try:
proxy.ping()
log.debug("located NS")
return proxy
except PyroError as x:
e = NamingError("Failed to locate the nameserver")
if sys.version_info >= (3, 0):
e.__cause__ = x
raise e