def test_static_servers(self):
broker = make_broker()
key_s = b'v0-1234-1'
servers_yaml = """\
storage:
v0-1234-1:
ann:
anonymous-storage-FURL: {furl}
permutation-seed-base32: aaaaaaaaaaaaaaaaaaaaaaaa
""".format(furl=SOME_FURL)
servers = yamlutil.safe_load(servers_yaml)
permseed = base32.a2b(b"aaaaaaaaaaaaaaaaaaaaaaaa")
broker.set_static_servers(servers["storage"])
self.failUnlessEqual(len(broker._static_server_ids), 1)
s = broker.servers[key_s]
self.failUnlessEqual(s.announcement,
servers["storage"]["v0-1234-1"]["ann"])
self.failUnlessEqual(s.get_serverid(), key_s)
self.assertEqual(s.get_permutation_seed(), permseed)
# if the Introducer announces the same thing, we're supposed to
# ignore it
ann2 = {
"service-name": "storage",
"anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(str(base32.b2a(b"1"), "utf-8")),
"permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb",
}
broker._got_announcement(key_s, ann2)
s2 = broker.servers[key_s]
self.assertIdentical(s2, s)
self.assertEqual(s2.get_permutation_seed(), permseed)
def test_static_servers(self):
broker = StorageFarmBroker(True, lambda h: Mock())
key_s = 'v0-1234-1'
servers_yaml = """\
storage:
v0-1234-1:
ann:
anonymous-storage-FURL: pb://ge@nowhere/fake
permutation-seed-base32: aaaaaaaaaaaaaaaaaaaaaaaa
"""
servers = yamlutil.safe_load(servers_yaml)
permseed = base32.a2b("aaaaaaaaaaaaaaaaaaaaaaaa")
broker.set_static_servers(servers["storage"])
self.failUnlessEqual(len(broker._static_server_ids), 1)
s = broker.servers[key_s]
self.failUnlessEqual(s.announcement,
servers["storage"]["v0-1234-1"]["ann"])
self.failUnlessEqual(s.get_serverid(), key_s)
self.assertEqual(s.get_permutation_seed(), permseed)
# if the Introducer announces the same thing, we're supposed to
# ignore it
ann2 = {
"service-name": "storage",
"anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(base32.b2a(str(1))),
"permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb",
}
broker._got_announcement(key_s, ann2)
s2 = broker.servers[key_s]
self.assertIdentical(s2, s)
self.assertEqual(s2.get_permutation_seed(), permseed)
def init_introducer_clients(self):
self.introducer_clients = []
self.introducer_furls = []
introducers_yaml_filename = os.path.join(self.basedir, "private", "introducers.yaml")
introducers_filepath = FilePath(introducers_yaml_filename)
try:
with introducers_filepath.open() as f:
introducers_yaml = yamlutil.safe_load(f)
introducers = introducers_yaml.get("introducers", {})
log.msg("found %d introducers in private/introducers.yaml" %
len(introducers))
except EnvironmentError:
introducers = {}
if "default" in introducers.keys():
raise ValueError("'default' introducer furl cannot be specified in introducers.yaml; please fix impossible configuration.")
# read furl from tahoe.cfg
tahoe_cfg_introducer_furl = self.get_config("client", "introducer.furl", None)
if tahoe_cfg_introducer_furl:
introducers[u'default'] = {'furl':tahoe_cfg_introducer_furl}
for petname, introducer in introducers.items():
introducer_cache_filepath = FilePath(os.path.join(self.basedir, "private", "introducer_{}_cache.yaml".format(petname)))
ic = IntroducerClient(self.tub, introducer['furl'],
self.nickname,
str(allmydata.__full_version__),
str(self.OLDEST_SUPPORTED_VERSION),
self.get_app_versions(), self._sequencer, introducer_cache_filepath)
self.introducer_clients.append(ic)
self.introducer_furls.append(introducer['furl'])
ic.setServiceParent(self)
def create_introducer_clients(config, main_tub):
"""
Read, validate and parse any 'introducers.yaml' configuration.
:returns: a list of IntroducerClient instances
"""
# we return this list
introducer_clients = []
introducers_yaml_filename = config.get_private_path("introducers.yaml")
introducers_filepath = FilePath(introducers_yaml_filename)
try:
with introducers_filepath.open() as f:
introducers_yaml = yamlutil.safe_load(f)
if introducers_yaml is None:
raise EnvironmentError(
EPERM,
"Can't read '{}'".format(introducers_yaml_filename),
introducers_yaml_filename,
)
introducers = introducers_yaml.get("introducers", {})
log.msg("found {} introducers in private/introducers.yaml".format(
len(introducers), ))
except EnvironmentError as e:
if e.errno != ENOENT:
raise
introducers = {}
if "default" in introducers.keys():
raise ValueError(
"'default' introducer furl cannot be specified in introducers.yaml;"
" please fix impossible configuration.")
# read furl from tahoe.cfg
tahoe_cfg_introducer_furl = config.get_config("client", "introducer.furl",
None)
if tahoe_cfg_introducer_furl == "None":
raise ValueError("tahoe.cfg has invalid 'introducer.furl = None':"
" to disable it, use 'introducer.furl ='"
" or omit the key entirely")
if tahoe_cfg_introducer_furl:
introducers[u'default'] = {'furl': tahoe_cfg_introducer_furl}
for petname, introducer in introducers.items():
introducer_cache_filepath = FilePath(
config.get_private_path(
"introducer_{}_cache.yaml".format(petname)))
ic = IntroducerClient(
main_tub,
introducer['furl'].encode("ascii"),
config.nickname,
str(allmydata.__full_version__),
str(_Client.OLDEST_SUPPORTED_VERSION),
node.get_app_versions(),
partial(_sequencer, config),
introducer_cache_filepath,
)
introducer_clients.append(ic)
return introducer_clients
def test_convert(self):
"""
Unicode and (ASCII) native strings get roundtripped to Unicode strings.
"""
data = yaml.safe_dump(
[six.ensure_str("str"), u"unicode", u"\u1234nicode"])
back = yamlutil.safe_load(data)
self.assertIsInstance(back[0], str)
self.assertIsInstance(back[1], str)
self.assertIsInstance(back[2], str)
def load_connections(self):
"""
Load the connections.yaml file if it exists, otherwise
create a default configuration.
"""
fn = os.path.join(self.basedir, "private", "connections.yaml")
connections_filepath = FilePath(fn)
try:
with connections_filepath.open() as f:
self.connections_config = yamlutil.safe_load(f)
except EnvironmentError:
self.connections_config = { 'servers' : {} }
content = yamlutil.safe_dump(self.connections_config)
connections_filepath.setContent(content)
def load_connections(self):
"""
Load the connections.yaml file if it exists, otherwise
create a default configuration.
"""
fn = os.path.join(self.basedir, "private", "connections.yaml")
connections_filepath = FilePath(fn)
try:
with connections_filepath.open() as f:
self.connections_config = yamlutil.safe_load(f)
except EnvironmentError:
self.connections_config = { 'servers' : {} }
content = yamlutil.safe_dump(self.connections_config)
connections_filepath.setContent(content)
def load_static_servers(self):
"""
Load the servers.yaml file if it exists, and provide the static
server data to the StorageFarmBroker.
"""
fn = os.path.join(self.basedir, "private", "servers.yaml")
servers_filepath = FilePath(fn)
try:
with servers_filepath.open() as f:
servers_yaml = yamlutil.safe_load(f)
static_servers = servers_yaml.get("storage", {})
log.msg("found %d static servers in private/servers.yaml" %
len(static_servers))
self.storage_broker.set_static_servers(static_servers)
except EnvironmentError:
pass
def load_static_servers(self):
"""
Load the servers.yaml file if it exists, and provide the static
server data to the StorageFarmBroker.
"""
fn = self.config.get_private_path("servers.yaml")
servers_filepath = FilePath(fn)
try:
with servers_filepath.open() as f:
servers_yaml = yamlutil.safe_load(f)
static_servers = servers_yaml.get("storage", {})
log.msg("found %d static servers in private/servers.yaml" %
len(static_servers))
self.storage_broker.set_static_servers(static_servers)
except EnvironmentError:
pass
def _load_announcements(self):
try:
with self._cache_filepath.open() as f:
servers = yamlutil.safe_load(f)
except EnvironmentError:
return # no cache file
if not isinstance(servers, list):
log.err(InvalidCacheError("not a list"), level=log.WEIRD)
return
self.log("Using server data from cache", level=log.UNUSUAL)
for server_params in servers:
if not isinstance(server_params, dict):
log.err(InvalidCacheError("not a dict: %r" % (server_params,)),
level=log.WEIRD)
continue
self._deliver_announcements(server_params['key_s'],
server_params['ann'])
def init_introducer_clients(self):
self.introducer_clients = []
self.introducer_furls = []
introducers_yaml_filename = os.path.join(self.basedir, "private",
"introducers.yaml")
introducers_filepath = FilePath(introducers_yaml_filename)
try:
with introducers_filepath.open() as f:
introducers_yaml = yamlutil.safe_load(f)
introducers = introducers_yaml.get("introducers", {})
log.msg("found %d introducers in private/introducers.yaml" %
len(introducers))
except EnvironmentError:
introducers = {}
if "default" in introducers.keys():
raise ValueError(
"'default' introducer furl cannot be specified in introducers.yaml; please fix impossible configuration."
)
# read furl from tahoe.cfg
tahoe_cfg_introducer_furl = self.get_config("client",
"introducer.furl", None)
if tahoe_cfg_introducer_furl == "None":
raise ValueError("tahoe.cfg has invalid 'introducer.furl = None':"
" to disable it, use 'introducer.furl ='"
" or omit the key entirely")
if tahoe_cfg_introducer_furl:
introducers[u'default'] = {'furl': tahoe_cfg_introducer_furl}
for petname, introducer in introducers.items():
introducer_cache_filepath = FilePath(
os.path.join(self.basedir, "private",
"introducer_{}_cache.yaml".format(petname)))
ic = IntroducerClient(self.tub, introducer['furl'].encode("ascii"),
self.nickname,
str(allmydata.__full_version__),
str(self.OLDEST_SUPPORTED_VERSION),
self.get_app_versions(), self._sequencer,
introducer_cache_filepath)
self.introducer_clients.append(ic)
self.introducer_furls.append(introducer['furl'])
ic.setServiceParent(self)
def _load_announcements(self):
try:
with self._cache_filepath.open() as f:
servers = yamlutil.safe_load(f)
except EnvironmentError:
return # no cache file
if not isinstance(servers, list):
log.err(InvalidCacheError("not a list"), level=log.WEIRD)
return
self.log("Using server data from cache", level=log.UNUSUAL)
for server_params in servers:
if not isinstance(server_params, dict):
log.err(InvalidCacheError("not a dict: %r" % (server_params,)),
level=log.WEIRD)
continue
# everything coming from yamlutil.safe_load is unicode
key_s = server_params['key_s'].encode("ascii")
self._deliver_announcements(key_s, server_params['ann'])
def create_introducer_clients(config, main_tub):
"""
Read, validate and parse any 'introducers.yaml' configuration.
:returns: a list of IntroducerClient instances
"""
# we return this list
introducer_clients = []
introducers_yaml_filename = config.get_private_path("introducers.yaml")
introducers_filepath = FilePath(introducers_yaml_filename)
try:
with introducers_filepath.open() as f:
introducers_yaml = yamlutil.safe_load(f)
if introducers_yaml is None:
raise EnvironmentError(
EPERM,
"Can't read '{}'".format(introducers_yaml_filename),
introducers_yaml_filename,
)
introducers = introducers_yaml.get("introducers", {})
log.msg(
"found {} introducers in private/introducers.yaml".format(
len(introducers),
)
)
except EnvironmentError as e:
if e.errno != ENOENT:
raise
introducers = {}
if "default" in introducers.keys():
raise ValueError(
"'default' introducer furl cannot be specified in introducers.yaml;"
" please fix impossible configuration."
)
# read furl from tahoe.cfg
tahoe_cfg_introducer_furl = config.get_config("client", "introducer.furl", None)
if tahoe_cfg_introducer_furl == "None":
raise ValueError(
"tahoe.cfg has invalid 'introducer.furl = None':"
" to disable it, use 'introducer.furl ='"
" or omit the key entirely"
)
if tahoe_cfg_introducer_furl:
introducers[u'default'] = {'furl':tahoe_cfg_introducer_furl}
for petname, introducer in introducers.items():
introducer_cache_filepath = FilePath(config.get_private_path("introducer_{}_cache.yaml".format(petname)))
ic = IntroducerClient(
main_tub,
introducer['furl'].encode("ascii"),
config.nickname,
str(allmydata.__full_version__),
str(_Client.OLDEST_SUPPORTED_VERSION),
node.get_app_versions(),
partial(_sequencer, config),
introducer_cache_filepath,
)
introducer_clients.append(ic)
return introducer_clients
def _load_cache(self, cache_filepath):
with cache_filepath.open() as f:
return yamlutil.safe_load(f)
def test_convert(self):
data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"])
back = yamlutil.safe_load(data)
self.assertIsInstance(back[0], str)
self.assertIsInstance(back[1], str)
self.assertIsInstance(back[2], str)
def _load_cache(self, cache_filepath):
with cache_filepath.open() as f:
return yamlutil.safe_load(f)
def load_magic_folders(node_directory):
"""
Loads existing magic-folder configuration and returns it as a dict
mapping name -> dict of config. This will NOT upgrade from
old-style to new-style config (but WILL read old-style config and
return in the same way as if it was new-style).
:returns: dict mapping magic-folder-name to its config (also a dict)
"""
yaml_fname = os.path.join(node_directory, u"private", u"magic_folders.yaml")
folders = dict()
config_fname = os.path.join(node_directory, "tahoe.cfg")
config = configutil.get_config(config_fname)
if not os.path.exists(yaml_fname):
# there will still be a magic_folder section in a "new"
# config, but it won't have local.directory nor poll_interval
# in it.
if config.has_option("magic_folder", "local.directory"):
up_fname = os.path.join(node_directory, "private", "magic_folder_dircap")
coll_fname = os.path.join(node_directory, "private", "collective_dircap")
directory = config.get("magic_folder", "local.directory").decode('utf8')
try:
interval = int(config.get("magic_folder", "poll_interval"))
except ConfigParser.NoOptionError:
interval = 60
dir_fp = to_filepath(directory)
if not dir_fp.exists():
raise Exception(
"The '[magic_folder] local.directory' parameter is {} "
"but there is no directory at that location.".format(
quote_local_unicode_path(directory),
)
)
if not dir_fp.isdir():
raise Exception(
"The '[magic_folder] local.directory' parameter is {} "
"but the thing at that location is not a directory.".format(
quote_local_unicode_path(directory)
)
)
folders[u"default"] = {
u"directory": directory,
u"upload_dircap": fileutil.read(up_fname),
u"collective_dircap": fileutil.read(coll_fname),
u"poll_interval": interval,
}
else:
# without any YAML file AND no local.directory option it's
# an error if magic-folder is "enabled" because we don't
# actually have enough config for any magic-folders at all
if config.has_section("magic_folder") \
and config.getboolean("magic_folder", "enabled") \
and not folders:
raise Exception(
"[magic_folder] is enabled but has no YAML file and no "
"'local.directory' option."
)
elif os.path.exists(yaml_fname): # yaml config-file exists
if config.has_option("magic_folder", "local.directory"):
raise Exception(
"magic-folder config has both old-style configuration"
" and new-style configuration; please remove the "
"'local.directory' key from tahoe.cfg or remove "
"'magic_folders.yaml' from {}".format(node_directory)
)
with open(yaml_fname, "r") as f:
magic_folders = yamlutil.safe_load(f.read())
if not isinstance(magic_folders, dict):
raise Exception(
"'{}' should contain a dict".format(yaml_fname)
)
folders = magic_folders['magic-folders']
if not isinstance(folders, dict):
raise Exception(
"'magic-folders' in '{}' should be a dict".format(yaml_fname)
)
# check configuration
for (name, mf_config) in folders.items():
if not isinstance(mf_config, dict):
raise Exception(
"Each item in '{}' must itself be a dict".format(yaml_fname)
)
for k in ['collective_dircap', 'upload_dircap', 'directory', 'poll_interval']:
if k not in mf_config:
raise Exception(
"Config for magic folder '{}' is missing '{}'".format(
name, k
)
)
for k in ['collective_dircap', 'upload_dircap']:
if isinstance(mf_config[k], unicode):
mf_config[k] = mf_config[k].encode('ascii')
return folders
def get_introducer_configuration(self):
"""
Get configuration for introducers.
:return {unicode: (unicode, FilePath)}: A mapping from introducer
petname to a tuple of the introducer's fURL and local cache path.
"""
introducers_yaml_filename = self.get_private_path("introducers.yaml")
introducers_filepath = FilePath(introducers_yaml_filename)
def get_cache_filepath(petname):
return FilePath(
self.get_private_path(
"introducer_{}_cache.yaml".format(petname)), )
try:
with introducers_filepath.open() as f:
introducers_yaml = safe_load(f)
if introducers_yaml is None:
raise EnvironmentError(
EPERM,
"Can't read '{}'".format(introducers_yaml_filename),
introducers_yaml_filename,
)
introducers = {
petname: config["furl"]
for petname, config in introducers_yaml.get(
"introducers", {}).items()
}
non_strs = list(k for k in introducers.keys()
if not isinstance(k, str))
if non_strs:
raise TypeError(
"Introducer petnames {!r} should have been str".format(
non_strs, ), )
non_strs = list(v for v in introducers.values()
if not isinstance(v, str))
if non_strs:
raise TypeError(
"Introducer fURLs {!r} should have been str".format(
non_strs, ), )
log.msg("found {} introducers in {!r}".format(
len(introducers),
introducers_yaml_filename,
))
except EnvironmentError as e:
if e.errno != ENOENT:
raise
introducers = {}
# supported the deprecated [client]introducer.furl item in tahoe.cfg
tahoe_cfg_introducer_furl = self.get_config("client",
"introducer.furl", None)
if tahoe_cfg_introducer_furl == "None":
raise ValueError("tahoe.cfg has invalid 'introducer.furl = None':"
" to disable it omit the key entirely")
if tahoe_cfg_introducer_furl:
warn(
"tahoe.cfg [client]introducer.furl is deprecated; "
"use private/introducers.yaml instead.",
category=DeprecationWarning,
stacklevel=-1,
)
if "default" in introducers:
raise ValueError(
"'default' introducer furl cannot be specified in tahoe.cfg and introducers.yaml;"
" please fix impossible configuration.")
introducers['default'] = tahoe_cfg_introducer_furl
return {
petname: (furl, get_cache_filepath(petname))
for (petname, furl) in introducers.items()
}
def test_convert(self):
data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"])
back = yamlutil.safe_load(data)
self.failUnlessEqual(type(back[0]), unicode)
self.failUnlessEqual(type(back[1]), unicode)
self.failUnlessEqual(type(back[2]), unicode)