def init_magic_folder(self):
#print "init_magic_folder"
if self.get_config("drop_upload", "enabled", False, boolean=True):
raise OldConfigOptionError("The [drop_upload] section must be renamed to [magic_folder].\n"
"See docs/frontends/magic-folder.rst for more information.")
if self.get_config("magic_folder", "enabled", False, boolean=True):
#print "magic folder enabled"
upload_dircap = self.get_private_config("magic_folder_dircap")
collective_dircap = self.get_private_config("collective_dircap")
local_dir_config = self.get_config("magic_folder", "local.directory").decode("utf-8")
local_dir = abspath_expanduser_unicode(local_dir_config, base=self.basedir)
dbfile = os.path.join(self.basedir, "private", "magicfolderdb.sqlite")
dbfile = abspath_expanduser_unicode(dbfile)
from allmydata.frontends import magic_folder
umask = self.get_config("magic_folder", "download.umask", 0077)
s = magic_folder.MagicFolder(self, upload_dircap, collective_dircap, local_dir, dbfile, umask)
self._magic_folder = s
s.setServiceParent(self)
s.startService()
# start processing the upload queue when we've connected to
# enough servers
threshold = min(self.encoding_params["k"],
self.encoding_params["happy"] + 1)
d = self.storage_broker.when_connected_enough(threshold)
d.addCallback(lambda ign: s.ready())
def test_web_static(self):
basedir = u"introducer.Node.test_web_static"
os.mkdir(basedir)
fileutil.write(os.path.join(basedir, "tahoe.cfg"),
"[node]\n" +
"web.port = tcp:0:interface=127.0.0.1\n" +
"web.static = relative\n")
c = IntroducerNode(basedir)
w = c.getServiceNamed("webish")
abs_basedir = fileutil.abspath_expanduser_unicode(basedir)
expected = fileutil.abspath_expanduser_unicode(u"relative", abs_basedir)
self.failUnlessReallyEqual(w.staticdir, expected)
def test_manhole_keyfile(self):
basedir = u"client.Basic.test_manhole_keyfile"
os.mkdir(basedir)
fileutil.write(os.path.join(basedir, "tahoe.cfg"),
BASECONFIG +
"[node]\n" +
"ssh.port = tcp:0:interface=127.0.0.1\n" +
"ssh.authorized_keys_file = relative\n")
c = client.Client(basedir)
m = [s for s in c if isinstance(s, AuthorizedKeysManhole)][0]
abs_basedir = fileutil.abspath_expanduser_unicode(basedir)
expected = fileutil.abspath_expanduser_unicode(u"relative", abs_basedir)
self.failUnlessReallyEqual(m.keyfile, expected)
def test_web_staticdir(self):
basedir = u"client.Basic.test_web_staticdir"
os.mkdir(basedir)
fileutil.write(os.path.join(basedir, "tahoe.cfg"),
BASECONFIG +
"[node]\n" +
"web.port = tcp:0:interface=127.0.0.1\n" +
"web.static = relative\n")
c = client.create_client(basedir)
w = c.getServiceNamed("webish")
abs_basedir = fileutil.abspath_expanduser_unicode(basedir)
expected = fileutil.abspath_expanduser_unicode(u"relative", abs_basedir)
self.failUnlessReallyEqual(w.staticdir, expected)
def setup_alice_and_bob(self, alice_clock=reactor, bob_clock=reactor):
self.set_up_grid(num_clients=2, oneshare=True)
self.alice_magicfolder = None
self.bob_magicfolder = None
alice_magic_dir = abspath_expanduser_unicode(u"Alice-magic", base=self.basedir)
self.mkdir_nonascii(alice_magic_dir)
bob_magic_dir = abspath_expanduser_unicode(u"Bob-magic", base=self.basedir)
self.mkdir_nonascii(bob_magic_dir)
# Alice creates a Magic Folder,
# invites herself then and joins.
d = self.do_create_magic_folder(0)
d.addCallback(lambda ign: self.do_invite(0, u"Alice\u00F8"))
def get_invite_code(result):
self.invite_code = result[1].strip()
d.addCallback(get_invite_code)
d.addCallback(lambda ign: self.do_join(0, alice_magic_dir, self.invite_code))
def get_alice_caps(ign):
self.alice_collective_dircap, self.alice_upload_dircap = self.get_caps_from_files(0)
d.addCallback(get_alice_caps)
d.addCallback(lambda ign: self.check_joined_config(0, self.alice_upload_dircap))
d.addCallback(lambda ign: self.check_config(0, alice_magic_dir))
def get_Alice_magicfolder(result):
self.alice_magicfolder = self.init_magicfolder(0, self.alice_upload_dircap,
self.alice_collective_dircap,
alice_magic_dir, alice_clock)
return result
d.addCallback(get_Alice_magicfolder)
# Alice invites Bob. Bob joins.
d.addCallback(lambda ign: self.do_invite(0, u"Bob\u00F8"))
def get_invite_code(result):
self.invite_code = result[1].strip()
d.addCallback(get_invite_code)
d.addCallback(lambda ign: self.do_join(1, bob_magic_dir, self.invite_code))
def get_bob_caps(ign):
self.bob_collective_dircap, self.bob_upload_dircap = self.get_caps_from_files(1)
d.addCallback(get_bob_caps)
d.addCallback(lambda ign: self.check_joined_config(1, self.bob_upload_dircap))
d.addCallback(lambda ign: self.check_config(1, bob_magic_dir))
def get_Bob_magicfolder(result):
self.bob_magicfolder = self.init_magicfolder(1, self.bob_upload_dircap,
self.bob_collective_dircap,
bob_magic_dir, bob_clock)
return result
d.addCallback(get_Bob_magicfolder)
return d
def init_magic_folder(self):
#print "init_magic_folder"
if self.get_config("drop_upload", "enabled", False, boolean=True):
raise OldConfigOptionError("The [drop_upload] section must be renamed to [magic_folder].\n"
"See docs/frontends/magic-folder.rst for more information.")
if self.get_config("magic_folder", "enabled", False, boolean=True):
from allmydata.frontends import magic_folder
try:
magic_folders = magic_folder.load_magic_folders(self.basedir)
except Exception as e:
log.msg("Error loading magic-folder config: {}".format(e))
raise
# start processing the upload queue when we've connected to
# enough servers
threshold = min(self.encoding_params["k"],
self.encoding_params["happy"] + 1)
for (name, mf_config) in magic_folders.items():
self.log("Starting magic_folder '{}'".format(name))
db_filename = os.path.join(self.basedir, "private", "magicfolder_{}.sqlite".format(name))
local_dir_config = mf_config['directory']
try:
poll_interval = int(mf_config["poll_interval"])
except ValueError:
raise ValueError("'poll_interval' option must be an int")
s = magic_folder.MagicFolder(
client=self,
upload_dircap=mf_config["upload_dircap"].encode('ascii'),
collective_dircap=mf_config["collective_dircap"].encode('ascii'),
local_path_u=abspath_expanduser_unicode(local_dir_config, base=self.basedir),
dbfile=abspath_expanduser_unicode(db_filename),
umask=self.get_config("magic_folder", "download.umask", 0077),
name=name,
downloader_delay=poll_interval,
)
self._magic_folders[name] = s
s.setServiceParent(self)
s.startService()
connected_d = self.storage_broker.when_connected_enough(threshold)
def connected_enough(ign, mf):
mf.ready() # returns a Deferred we ignore
return None
connected_d.addCallback(connected_enough, s)
def __init__(self, config, basedir=u"."):
"""
Initialize the node with the given configuration. It's base directory
is the current directory by default.
"""
service.MultiService.__init__(self)
# ideally, this would only be in _Config (or otherwise abstracted)
self.basedir = abspath_expanduser_unicode(unicode(basedir))
# XXX don't write files in ctor!
fileutil.make_dirs(os.path.join(self.basedir, "private"), 0700)
with open(os.path.join(self.basedir, "private", "README"), "w") as f:
f.write(PRIV_README)
self.config = config
self.get_config = config.get_config # XXX stopgap
self.nickname = config.nickname # XXX stopgap
self.init_tempdir()
self.check_privacy()
self.create_log_tub()
self.logSource = "Node"
self.setup_logging()
self.create_i2p_provider()
self.create_tor_provider()
self.init_connections()
self.set_tub_options()
self.create_main_tub()
self.create_control_tub()
self.log("Node constructed. " + get_package_versions_string())
iputil.increase_rlimits()
def test_join_twice_failure(self):
self.basedir = "cli/MagicFolder/create-join-twice-failure"
os.makedirs(self.basedir)
self.set_up_grid(oneshare=True)
local_dir = os.path.join(self.basedir, "magic")
abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir), long_path=False)
d = self.do_create_magic_folder(0)
d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname))
def get_invite_code_and_join((rc, stdout, stderr)):
self.invite_code = stdout.strip()
return self.do_join(0, unicode(local_dir), self.invite_code)
d.addCallback(get_invite_code_and_join)
def get_caps(ign):
self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0)
d.addCallback(get_caps)
d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap))
d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u))
def join_again(ignore):
return self.do_cli("magic-folder", "join", self.invite_code, local_dir, client_num=0)
d.addCallback(join_again)
def get_results(result):
(rc, out, err) = result
self.failUnlessEqual(out, "")
self.failUnlessIn("This client has already joined a magic folder.", err)
self.failUnlessIn("Use the 'tahoe magic-folder leave' command first.", err)
self.failIfEqual(rc, 0)
d.addCallback(get_results)
return d
def test_unlinked_immutable_from_file(self):
# tahoe put file.txt
# tahoe put ./file.txt
# tahoe put /tmp/file.txt
# tahoe put ~/file.txt
self.basedir = "cli/Put/unlinked_immutable_from_file"
self.set_up_grid(oneshare=True)
rel_fn = os.path.join(self.basedir, "DATAFILE")
abs_fn = unicode_to_argv(abspath_expanduser_unicode(unicode(rel_fn)))
# we make the file small enough to fit in a LIT file, for speed
fileutil.write(rel_fn, "short file")
d = self.do_cli("put", rel_fn)
def _uploaded((rc, out, err)):
readcap = out
self.failUnless(readcap.startswith("URI:LIT:"), readcap)
self.readcap = readcap
d.addCallback(_uploaded)
d.addCallback(lambda res: self.do_cli("put", "./" + rel_fn))
d.addCallback(lambda (rc,stdout,stderr):
self.failUnlessReallyEqual(stdout, self.readcap))
d.addCallback(lambda res: self.do_cli("put", abs_fn))
d.addCallback(lambda (rc,stdout,stderr):
self.failUnlessReallyEqual(stdout, self.readcap))
# we just have to assume that ~ is handled properly
return d
def test_error_on_old_config_files(self, mock_log_msg):
basedir = "test_client.Basic.test_error_on_old_config_files"
os.mkdir(basedir)
fileutil.write(os.path.join(basedir, "tahoe.cfg"),
BASECONFIG +
"[storage]\n" +
"enabled = false\n" +
"reserved_space = bogus\n")
fileutil.write(os.path.join(basedir, "introducer.furl"), "")
fileutil.write(os.path.join(basedir, "no_storage"), "")
fileutil.write(os.path.join(basedir, "readonly_storage"), "")
fileutil.write(os.path.join(basedir, "debug_discard_storage"), "")
e = self.failUnlessRaises(OldConfigError, client.Client, basedir)
abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)).encode(sys.getfilesystemencoding())
self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0])
self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0])
self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0])
self.failUnlessIn(os.path.join(abs_basedir, "debug_discard_storage"), e.args[0])
for oldfile in ['introducer.furl', 'no_storage', 'readonly_storage',
'debug_discard_storage']:
logged = [ m for m in mock_log_msg.call_args_list if
("Found pre-Tahoe-LAFS-v1.3 configuration file" in str(m[0][0]) and oldfile in str(m[0][0])) ]
self.failUnless(logged, (oldfile, mock_log_msg.call_args_list))
for oldfile in [
'nickname', 'webport', 'keepalive_timeout', 'log_gatherer.furl',
'disconnect_timeout', 'advertised_ip_addresses', 'helper.furl',
'key_generator.furl', 'stats_gatherer.furl', 'sizelimit',
'run_helper']:
logged = [ m for m in mock_log_msg.call_args_list if
("Found pre-Tahoe-LAFS-v1.3 configuration file" in str(m[0][0]) and oldfile in str(m[0][0])) ]
self.failIf(logged, oldfile)
def test_join_leave_join(self):
self.basedir = "cli/MagicFolder/create-join-leave-join"
os.makedirs(self.basedir)
self.set_up_grid(oneshare=True)
local_dir = os.path.join(self.basedir, "magic")
abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir), long_path=False)
self.invite_code = None
d = self.do_create_magic_folder(0)
d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname))
def get_invite_code_and_join((rc, stdout, stderr)):
self.failUnlessEqual(rc, 0)
self.invite_code = stdout.strip()
return self.do_join(0, unicode(local_dir), self.invite_code)
d.addCallback(get_invite_code_and_join)
def get_caps(ign):
self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0)
d.addCallback(get_caps)
d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap))
d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u))
d.addCallback(lambda ign: self.do_leave(0))
d.addCallback(lambda ign: self.do_join(0, unicode(local_dir), self.invite_code))
def get_caps(ign):
self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0)
d.addCallback(get_caps)
d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap))
d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u))
return d
def test_private_config(self):
basedir = u"test_node/test_private_config"
privdir = os.path.join(basedir, "private")
fileutil.make_dirs(privdir)
f = open(os.path.join(privdir, 'already'), 'wt')
f.write("secret")
f.close()
basedir = fileutil.abspath_expanduser_unicode(basedir)
config = config_from_string(basedir, "", "")
self.assertEqual(config.get_private_config("already"), "secret")
self.assertEqual(config.get_private_config("not", "default"), "default")
self.assertRaises(MissingConfigEntry, config.get_private_config, "not")
value = config.get_or_create_private_config("new", "start")
self.assertEqual(value, "start")
self.assertEqual(config.get_private_config("new"), "start")
counter = []
def make_newer():
counter.append("called")
return "newer"
value = config.get_or_create_private_config("newer", make_newer)
self.assertEqual(len(counter), 1)
self.assertEqual(value, "newer")
self.assertEqual(config.get_private_config("newer"), "newer")
value = config.get_or_create_private_config("newer", make_newer)
self.assertEqual(len(counter), 1) # don't call unless necessary
self.assertEqual(value, "newer")
def run(self):
options = self.options
nodeurl = options['node-url']
self.verbosity = 1
if options['quiet']:
self.verbosity = 0
if options['verbose']:
self.verbosity = 2
stdout = options.stdout
stderr = options.stderr
start_timestamp = datetime.datetime.now()
self.backupdb = None
bdbfile = os.path.join(options["node-directory"],
"private", "backupdb.sqlite")
bdbfile = abspath_expanduser_unicode(bdbfile)
self.backupdb = backupdb.get_backupdb(bdbfile, stderr)
if not self.backupdb:
print >>stderr, "ERROR: Unable to load backup db."
return 1
try:
rootcap, path = get_alias(options.aliases, options.to_dir, DEFAULT_ALIAS)
except UnknownAliasError, e:
e.display(stderr)
return 1
def __init__(self, basedir=u"."):
service.MultiService.__init__(self)
self.basedir = abspath_expanduser_unicode(unicode(basedir))
self.config_fname = os.path.join(self.basedir, "tahoe.cfg")
self._portnumfile = os.path.join(self.basedir, self.PORTNUMFILE)
fileutil.make_dirs(os.path.join(self.basedir, "private"), 0700)
with open(os.path.join(self.basedir, "private", "README"), "w") as f:
f.write(PRIV_README)
# creates self.config
self.read_config()
nickname_utf8 = self.get_config("node", "nickname", "<unspecified>")
self.nickname = nickname_utf8.decode("utf-8")
assert type(self.nickname) is unicode
self.init_tempdir()
self.check_privacy()
self.create_log_tub()
self.logSource="Node"
self.setup_logging()
self.create_i2p_provider()
self.create_tor_provider()
self.init_connections()
self.set_tub_options()
self.create_main_tub()
self.create_control_tub()
self.log("Node constructed. " + get_package_versions_string())
iputil.increase_rlimits()
def __init__(self, basedir, mode):
self.basedir = basedir = abspath_expanduser_unicode(unicode(basedir))
if not (basedir + os.path.sep).startswith(abspath_expanduser_unicode(u".") + os.path.sep):
raise AssertionError("safety issue: basedir must be a subdir")
self.testdir = testdir = os.path.join(basedir, "test")
if os.path.exists(testdir):
shutil.rmtree(testdir)
fileutil.make_dirs(testdir)
self.sparent = service.MultiService()
self.sparent.startService()
self.proc = None
self.tub = Tub()
self.tub.setOption("expose-remote-exception-types", False)
self.tub.setServiceParent(self.sparent)
self.mode = mode
self.failed = False
self.keepalive_file = None
def get_source_info(self, source_spec):
"""
This turns an argv string into a (Local|Tahoe)(File|Directory)Source.
"""
precondition(isinstance(source_spec, unicode), source_spec)
rootcap, path_utf8 = get_alias(self.aliases, source_spec, None)
path = path_utf8.decode("utf-8")
# any trailing slash is removed in abspath_expanduser_unicode(), so
# make a note of it here, to throw an error later
had_trailing_slash = path.endswith("/")
if rootcap == DefaultAliasMarker:
# no alias, so this is a local file
pathname = abspath_expanduser_unicode(path)
name = os.path.basename(pathname)
if not os.path.exists(pathname):
raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path)
if os.path.isdir(pathname):
t = LocalDirectorySource(self.progress, pathname, name)
else:
if had_trailing_slash:
raise FilenameWithTrailingSlashError(source_spec,
quotefn=quote_local_unicode_path)
if not os.path.isfile(pathname):
raise WeirdSourceError(pathname)
t = LocalFileSource(pathname, name) # non-empty
else:
# this is a tahoe object
url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
name = None
if path:
if path.endswith("/"):
path = path[:-1]
url += "/" + escape_path(path)
last_slash = path.rfind(u"/")
name = path
if last_slash != -1:
name = path[last_slash+1:]
resp = do_http("GET", url + "?t=json")
if resp.status == 404:
raise MissingSourceError(source_spec)
elif resp.status != 200:
raise HTTPError("Error examining source %s" % quote_output(source_spec),
resp)
parsed = json.loads(resp.read())
nodetype, d = parsed
if nodetype == "dirnode":
t = TahoeDirectorySource(self.nodeurl, self.cache,
self.progress, name)
t.init_from_parsed(parsed)
else:
if had_trailing_slash:
raise FilenameWithTrailingSlashError(source_spec)
writecap = to_str(d.get("rw_uri"))
readcap = to_str(d.get("ro_uri"))
mutable = d.get("mutable", False) # older nodes don't provide it
t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name)
return t
def setup_ssh(self):
ssh_port = self.get_config("node", "ssh.port", "")
if ssh_port:
ssh_keyfile_config = self.get_config("node", "ssh.authorized_keys_file").decode('utf-8')
ssh_keyfile = abspath_expanduser_unicode(ssh_keyfile_config, base=self.basedir)
from allmydata import manhole
m = manhole.AuthorizedKeysManhole(ssh_port, ssh_keyfile)
m.setServiceParent(self)
self.log("AuthorizedKeysManhole listening on %s" % (ssh_port,))
def init_web(self, webport):
self.log("init_web(webport=%s)", args=(webport,), umid="2bUygA")
from allmydata.webish import IntroducerWebishServer
nodeurl_path = os.path.join(self.basedir, u"node.url")
config_staticdir = self.get_config("node", "web.static", "public_html").decode('utf-8')
staticdir = abspath_expanduser_unicode(config_staticdir, base=self.basedir)
ws = IntroducerWebishServer(self, webport, nodeurl_path, staticdir)
self.add_service(ws)
def init_web(self, webport):
self.log("init_web(webport=%s)", args=(webport,))
from allmydata.webish import WebishServer
nodeurl_path = os.path.join(self.basedir, "node.url")
staticdir_config = self.get_config("node", "web.static", "public_html").decode("utf-8")
staticdir = abspath_expanduser_unicode(staticdir_config, base=self.basedir)
ws = WebishServer(self, webport, nodeurl_path, staticdir)
self.add_service(ws)
def init_magicfolder(self, client_num, upload_dircap, collective_dircap, local_magic_dir, clock):
dbfile = abspath_expanduser_unicode(u"magicfolderdb.sqlite", base=self.get_clientdir(i=client_num))
magicfolder = MagicFolder(self.get_client(client_num), upload_dircap, collective_dircap, local_magic_dir,
dbfile, 0077, pending_delay=0.2, clock=clock)
magicfolder.downloader._turn_delay = 0
magicfolder.setServiceParent(self.get_client(client_num))
magicfolder.ready()
return magicfolder
def process(self, event):
event_filepath_u = event.src_path.decode(encodingutil.get_filesystem_encoding())
event_filepath_u = abspath_expanduser_unicode(event_filepath_u, base=self._path)
if event_filepath_u == self._path:
# ignore events for parent directory
return
self._maybe_notify(event_filepath_u, event)
def argv_to_abspath(s, **kwargs):
"""
Convenience function to decode an argv element to an absolute path, with ~ expanded.
If this fails, raise a UsageError.
"""
decoded = argv_to_unicode(s)
if decoded.startswith(u'-'):
raise usage.UsageError("Path argument %s cannot start with '-'.\nUse %s if you intended to refer to a file."
% (quote_output(s), quote_output(os.path.join('.', s))))
return abspath_expanduser_unicode(decoded, **kwargs)
def get_source_info(self, source_spec):
"""
This turns an argv string into a (Local|Tahoe)(File|Directory)Source.
"""
precondition(isinstance(source_spec, unicode), source_spec)
rootcap, path_utf8 = get_alias(self.aliases, source_spec, None)
path = path_utf8.decode("utf-8")
if rootcap == DefaultAliasMarker:
# no alias, so this is a local file
pathname = abspath_expanduser_unicode(path)
name = os.path.basename(pathname)
if not os.path.exists(pathname):
raise MissingSourceError(source_spec, quotefn=quote_local_unicode_path)
if os.path.isdir(pathname):
t = LocalDirectorySource(self.progress, pathname, name)
else:
assert os.path.isfile(pathname)
t = LocalFileSource(pathname, name) # non-empty
else:
# this is a tahoe object
url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
name = None
if path:
url += "/" + escape_path(path)
last_slash = path.rfind(u"/")
name = path
if last_slash != -1:
name = path[last_slash+1:]
resp = do_http("GET", url + "?t=json")
if resp.status == 404:
raise MissingSourceError(source_spec)
elif resp.status != 200:
raise HTTPError("Error examining source %s" % quote_output(source_spec),
resp)
parsed = simplejson.loads(resp.read())
nodetype, d = parsed
if nodetype == "dirnode":
t = TahoeDirectorySource(self.nodeurl, self.cache,
self.progress, name)
t.init_from_parsed(parsed)
else:
writecap = to_str(d.get("rw_uri"))
readcap = to_str(d.get("ro_uri"))
mutable = d.get("mutable", False) # older nodes don't provide it
last_slash = source_spec.rfind(u"/")
if last_slash != -1:
# TODO: this looks funny and redundant with the 'name'
# assignment above. cf #2329
name = source_spec[last_slash+1:]
t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap, name)
return t
def get_config_path(self, *args):
"""
returns an absolute path inside the config directory with any
extra args join()-ed
"""
# note: we re-expand here (_basedir already went through this
# expanduser function) in case the path we're being asked for
# has embedded ".."'s in it
return abspath_expanduser_unicode(
os.path.join(self._basedir, *args)
)
def init_tempdir(self):
tempdir_config = self.get_config("node", "tempdir", "tmp").decode('utf-8')
tempdir = abspath_expanduser_unicode(tempdir_config, base=self.basedir)
if not os.path.exists(tempdir):
fileutil.make_dirs(tempdir)
tempfile.tempdir = tempdir
# this should cause twisted.web.http (which uses
# tempfile.TemporaryFile) to put large request bodies in the given
# directory. Without this, the default temp dir is usually /tmp/,
# which is frequently too small.
test_name = tempfile.mktemp()
_assert(os.path.dirname(test_name) == tempdir, test_name, tempdir)
def setUp(self):
yield super(StatusMagicFolder, self).setUp()
self.basedir="mf_list"
self.set_up_grid(oneshare=True)
self.local_dir = os.path.join(self.basedir, "magic")
os.mkdir(self.local_dir)
self.abs_local_dir_u = abspath_expanduser_unicode(unicode(self.local_dir), long_path=False)
yield self.do_create_magic_folder(0)
(rc, stdout, stderr) = yield self.do_invite(0, self.alice_nickname)
invite_code = stdout.strip()
yield self.do_join(0, unicode(self.local_dir), invite_code)
def init_ftp_server(self):
if self.get_config("ftpd", "enabled", False, boolean=True):
accountfile = from_utf8_or_none(self.get_config("ftpd", "accounts.file", None))
if accountfile:
accountfile = abspath_expanduser_unicode(accountfile, base=self.basedir)
accounturl = self.get_config("ftpd", "accounts.url", None)
ftp_portstr = self.get_config("ftpd", "port", "8021")
from allmydata.frontends import ftpd
s = ftpd.FTPServer(self, accountfile, accounturl, ftp_portstr)
s.setServiceParent(self)
def test_exclude_from_tilde_expansion(self, mock):
basedir = "cli/Backup/exclude_from_tilde_expansion"
fileutil.make_dirs(basedir)
nodeurl_path = os.path.join(basedir, 'node.url')
fileutil.write(nodeurl_path, 'http://example.net:2357/')
def parse(args): return parse_options(basedir, "backup", args)
# ensure that tilde expansion is performed on exclude-from argument
exclude_file = u'~/.tahoe/excludes.dummy'
mock.return_value = StringIO()
parse(['--exclude-from', unicode_to_argv(exclude_file), 'from', 'to'])
self.failUnlessIn(((abspath_expanduser_unicode(exclude_file),), {}), mock.call_args_list)
def test_create_invite_join(self):
self.basedir = "cli/MagicFolder/create-invite-join"
self.set_up_grid(oneshare=True)
local_dir = os.path.join(self.basedir, "magic")
abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir), long_path=False)
d = self.do_cli("magic-folder", "create", "magic:", "Alice", local_dir)
def _done((rc, stdout, stderr)):
self.failUnlessEqual(rc, 0)
self.collective_dircap, self.upload_dircap = self.get_caps_from_files(0)
d.addCallback(_done)
d.addCallback(lambda ign: self.check_joined_config(0, self.upload_dircap))
d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u))
return d
def init_sftp_server(self):
if self.get_config("sftpd", "enabled", False, boolean=True):
accountfile = from_utf8_or_none(self.get_config("sftpd", "accounts.file", None))
if accountfile:
accountfile = abspath_expanduser_unicode(accountfile, base=self.basedir)
accounturl = self.get_config("sftpd", "accounts.url", None)
sftp_portstr = self.get_config("sftpd", "port", "8022")
pubkey_file = from_utf8_or_none(self.get_config("sftpd", "host_pubkey_file"))
privkey_file = from_utf8_or_none(self.get_config("sftpd", "host_privkey_file"))
from allmydata.frontends import sftpd
s = sftpd.SFTPServer(self, accountfile, accounturl, sftp_portstr, pubkey_file, privkey_file)
s.setServiceParent(self)
def get_source_info(self, source_spec):
rootcap, path = get_alias(self.aliases, source_spec, None)
if rootcap == DefaultAliasMarker:
# no alias, so this is a local file
pathname = abspath_expanduser_unicode(path.decode('utf-8'))
name = os.path.basename(pathname)
if not os.path.exists(pathname):
raise MissingSourceError(source_spec)
if os.path.isdir(pathname):
t = LocalDirectorySource(self.progress, pathname)
else:
assert os.path.isfile(pathname)
t = LocalFileSource(pathname) # non-empty
else:
# this is a tahoe object
url = self.nodeurl + "uri/%s" % urllib.quote(rootcap)
name = None
if path:
url += "/" + escape_path(path)
last_slash = path.rfind("/")
name = path
if last_slash:
name = path[last_slash+1:]
resp = do_http("GET", url + "?t=json")
if resp.status == 404:
raise MissingSourceError(source_spec)
elif resp.status != 200:
raise HTTPError("Error examining source %s" % quote_output(source_spec),
resp)
parsed = simplejson.loads(resp.read())
nodetype, d = parsed
if nodetype == "dirnode":
t = TahoeDirectorySource(self.nodeurl, self.cache,
self.progress)
t.init_from_parsed(parsed)
else:
writecap = to_str(d.get("rw_uri"))
readcap = to_str(d.get("ro_uri"))
mutable = d.get("mutable", False) # older nodes don't provide it
if source_spec.rfind('/') != -1:
name = source_spec[source_spec.rfind('/')+1:]
t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap)
return name, t
def read_config(basedir, portnumfile, generated_files=[], _valid_config=None):
"""
Read and validate configuration.
:param unicode basedir: directory where configuration data begins
:param unicode portnumfile: filename fragment for "port number" files
:param list generated_files: a list of automatically-generated
configuration files.
:param ValidConfiguration _valid_config: (internal use, optional) a
structure defining valid configuration sections and keys
:returns: :class:`allmydata.node._Config` instance
"""
basedir = abspath_expanduser_unicode(ensure_text(basedir))
if _valid_config is None:
_valid_config = _common_valid_config()
# complain if there's bad stuff in the config dir
_error_about_old_config_files(basedir, generated_files)
# canonicalize the portnum file
portnumfile = os.path.join(basedir, portnumfile)
config_path = FilePath(basedir).child("tahoe.cfg")
try:
config_str = config_path.getContent()
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
# The file is missing, just create empty ConfigParser.
config_str = u""
else:
config_str = config_str.decode("utf-8-sig")
return config_from_string(
basedir,
portnumfile,
config_str,
_valid_config,
config_path,
)
def get_target_info(self, destination_spec):
precondition(isinstance(destination_spec, str), destination_spec)
rootcap, path_utf8 = get_alias(self.aliases, destination_spec, None)
path = path_utf8.decode("utf-8")
if rootcap == DefaultAliasMarker:
# no alias, so this is a local file
pathname = abspath_expanduser_unicode(path)
if not os.path.exists(pathname):
t = LocalMissingTarget(pathname)
elif os.path.isdir(pathname):
t = LocalDirectoryTarget(self.progress, pathname)
else:
# TODO: should this be _assert? what happens if the target is
# a special file?
assert os.path.isfile(pathname), pathname
t = LocalFileTarget(pathname) # non-empty
else:
# this is a tahoe object
url = self.nodeurl + "uri/%s" % url_quote(rootcap)
if path:
url += "/" + escape_path(path)
resp = do_http("GET", url + "?t=json")
if resp.status == 404:
# doesn't exist yet
t = TahoeMissingTarget(url)
elif resp.status == 200:
parsed = json.loads(resp.read())
nodetype, d = parsed
if nodetype == "dirnode":
t = TahoeDirectoryTarget(self.nodeurl, self.cache,
self.progress)
t.init_from_parsed(parsed)
else:
writecap = to_bytes(d.get("rw_uri"))
readcap = to_bytes(d.get("ro_uri"))
mutable = d.get("mutable", False)
t = TahoeFileTarget(self.nodeurl, mutable, writecap,
readcap, url)
else:
raise HTTPError(
"Error examining target %s" %
quote_output(destination_spec), resp)
return t
def test_join_twice_failure(self):
self.basedir = "cli/MagicFolder/create-join-twice-failure"
os.makedirs(self.basedir)
self.set_up_grid(oneshare=True)
local_dir = os.path.join(self.basedir, "magic")
abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir),
long_path=False)
d = self.do_create_magic_folder(0)
d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname))
def get_invite_code_and_join(args):
(rc, stdout, stderr) = args
self.invite_code = stdout.strip()
return self.do_join(0, unicode(local_dir), self.invite_code)
d.addCallback(get_invite_code_and_join)
def get_caps(ign):
self.collective_dircap, self.upload_dircap = self.get_caps_from_files(
0)
d.addCallback(get_caps)
d.addCallback(
lambda ign: self.check_joined_config(0, self.upload_dircap))
d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u))
def join_again(ignore):
return self.do_cli("magic-folder",
"join",
self.invite_code,
local_dir,
client_num=0)
d.addCallback(join_again)
def get_results(result):
(rc, out, err) = result
self.failUnlessEqual(out, "")
self.failUnlessIn("This client already has a magic-folder", err)
self.failIfEqual(rc, 0)
d.addCallback(get_results)
return d
def test_error_on_old_config_files(self):
basedir = "test_client.Basic.test_error_on_old_config_files"
os.mkdir(basedir)
fileutil.write(os.path.join(basedir, "tahoe.cfg"),
BASECONFIG +
"[storage]\n" +
"enabled = false\n" +
"reserved_space = bogus\n")
fileutil.write(os.path.join(basedir, "introducer.furl"), "")
fileutil.write(os.path.join(basedir, "no_storage"), "")
fileutil.write(os.path.join(basedir, "readonly_storage"), "")
fileutil.write(os.path.join(basedir, "debug_discard_storage"), "")
logged_messages = []
self.patch(twisted.python.log, 'msg', logged_messages.append)
e = self.failUnlessRaises(
OldConfigError,
read_config,
basedir,
"client.port",
_valid_config_sections=client._valid_config_sections,
)
abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)).encode(sys.getfilesystemencoding())
self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0])
self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0])
self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0])
self.failUnlessIn(os.path.join(abs_basedir, "debug_discard_storage"), e.args[0])
for oldfile in ['introducer.furl', 'no_storage', 'readonly_storage',
'debug_discard_storage']:
logged = [ m for m in logged_messages if
("Found pre-Tahoe-LAFS-v1.3 configuration file" in str(m) and oldfile in str(m)) ]
self.failUnless(logged, (oldfile, logged_messages))
for oldfile in [
'nickname', 'webport', 'keepalive_timeout', 'log_gatherer.furl',
'disconnect_timeout', 'advertised_ip_addresses', 'helper.furl',
'key_generator.furl', 'stats_gatherer.furl', 'sizelimit',
'run_helper']:
logged = [ m for m in logged_messages if
("Found pre-Tahoe-LAFS-v1.3 configuration file" in str(m) and oldfile in str(m)) ]
self.failIf(logged, (oldfile, logged_messages))
def test_join_leave_join(self):
self.basedir = "cli/MagicFolder/create-join-leave-join"
os.makedirs(self.basedir)
self.set_up_grid(oneshare=True)
local_dir = os.path.join(self.basedir, "magic")
abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir),
long_path=False)
self.invite_code = None
d = self.do_create_magic_folder(0)
d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname))
def get_invite_code_and_join(args):
(rc, stdout, stderr) = args
self.failUnlessEqual(rc, 0)
self.invite_code = stdout.strip()
return self.do_join(0, unicode(local_dir), self.invite_code)
d.addCallback(get_invite_code_and_join)
def get_caps(ign):
self.collective_dircap, self.upload_dircap = self.get_caps_from_files(
0)
d.addCallback(get_caps)
d.addCallback(
lambda ign: self.check_joined_config(0, self.upload_dircap))
d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u))
d.addCallback(lambda ign: self.do_leave(0))
d.addCallback(
lambda ign: self.do_join(0, unicode(local_dir), self.invite_code))
def get_caps(ign):
self.collective_dircap, self.upload_dircap = self.get_caps_from_files(
0)
d.addCallback(get_caps)
d.addCallback(
lambda ign: self.check_joined_config(0, self.upload_dircap))
d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u))
return d
def test_create_invite_join(self):
self.basedir = "cli/MagicFolder/create-invite-join"
self.set_up_grid(oneshare=True)
local_dir = os.path.join(self.basedir, "magic")
abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir),
long_path=False)
d = self.do_cli("magic-folder", "create", "magic:", "Alice", local_dir)
def _done((rc, stdout, stderr)):
self.failUnlessEqual(rc, 0)
self.collective_dircap, self.upload_dircap = self.get_caps_from_files(
0)
d.addCallback(_done)
d.addCallback(
lambda ign: self.check_joined_config(0, self.upload_dircap))
d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u))
return d
def __init__(self, client, accountfile):
self.client = client
self.passwords = {}
self.pubkeys = {}
self.rootcaps = {}
for line in open(abspath_expanduser_unicode(accountfile), "r"):
line = line.strip()
if line.startswith("#") or not line:
continue
name, passwd, rest = line.split(None, 2)
if passwd.startswith("ssh-"):
bits = rest.split()
keystring = " ".join([passwd] + bits[:-1])
rootcap = bits[-1]
self.pubkeys[name] = keystring
else:
self.passwords[name] = passwd
rootcap = rest
self.rootcaps[name] = rootcap
def init_magicfolder(self, client_num, upload_dircap, collective_dircap,
local_magic_dir, clock):
dbfile = abspath_expanduser_unicode(
u"magicfolderdb.sqlite", base=self.get_clientdir(i=client_num))
magicfolder = MagicFolder(
client=self.get_client(client_num),
upload_dircap=upload_dircap,
collective_dircap=collective_dircap,
local_path_u=local_magic_dir,
dbfile=dbfile,
umask=0o077,
pending_delay=0.2,
clock=clock,
)
magicfolder.downloader._turn_delay = 0
magicfolder.setServiceParent(self.get_client(client_num))
magicfolder.ready()
return magicfolder
def __init__(self, configparser, portnum_fname, basedir, config_fname):
"""
:param configparser: a ConfigParser instance
:param portnum_fname: filename to use for the port-number file
(a relative path inside basedir)
:param basedir: path to our "node directory", inside which all
configuration is managed
:param config_fname: the pathname actually used to create the
configparser (might be 'fake' if using in-memory data)
"""
self.portnum_fname = portnum_fname
self._basedir = abspath_expanduser_unicode(ensure_text(basedir))
self._config_fname = config_fname
self.config = configparser
self.nickname = self.get_config("node", "nickname", u"<unspecified>")
assert isinstance(self.nickname, str)
def read_config(basedir, portnumfile, generated_files=[], _valid_config=None):
"""
Read and validate configuration.
:param unicode basedir: directory where configuration data begins
:param unicode portnumfile: filename fragment for "port number" files
:param list generated_files: a list of automatically-generated
configuration files.
:param ValidConfiguration _valid_config: (internal use, optional) a
structure defining valid configuration sections and keys
:returns: :class:`allmydata.node._Config` instance
"""
basedir = abspath_expanduser_unicode(ensure_text(basedir))
if _valid_config is None:
_valid_config = _common_valid_config()
# complain if there's bad stuff in the config dir
_error_about_old_config_files(basedir, generated_files)
# canonicalize the portnum file
portnumfile = os.path.join(basedir, portnumfile)
# (try to) read the main config file
config_fname = os.path.join(basedir, "tahoe.cfg")
try:
parser = configutil.get_config(config_fname)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
# The file is missing, just create empty ConfigParser.
parser = configutil.get_config_from_string(u"")
configutil.validate_config(config_fname, parser, _valid_config)
# make sure we have a private configuration area
fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
return _Config(parser, portnumfile, basedir, config_fname)
def __init__(self, basedir=u"."):
service.MultiService.__init__(self)
self.basedir = abspath_expanduser_unicode(unicode(basedir))
self._portnumfile = os.path.join(self.basedir, self.PORTNUMFILE)
fileutil.make_dirs(os.path.join(self.basedir, "private"), 0700)
open(os.path.join(self.basedir, "private", "README"),
"w").write(PRIV_README)
# creates self.config
self.read_config()
nickname_utf8 = self.get_config("node", "nickname", "<unspecified>")
self.nickname = nickname_utf8.decode("utf-8")
assert type(self.nickname) is unicode
self.init_tempdir()
self.create_tub()
self.logSource = "Node"
self.setup_logging()
self.log("Node constructed. " + get_package_versions_string())
iputil.increase_rlimits()
def test_absolute_storage_dir(self):
"""
If the ``storage_dir`` item in the ``storage`` section of the
configuration gives an absolute path then exactly that path is used.
"""
basedir = u"client.Basic.test_absolute_storage_dir"
# create_client is going to try to make the storage directory so we
# don't want a literal absolute path like /myowndir which we won't
# have write permission to. So construct an absolute path that we
# should be able to write to.
base = u"\N{SNOWMAN}"
if encodingutil.filesystem_encoding != "utf-8":
base = u"melted_snowman"
expected_path = abspath_expanduser_unicode(
u"client.Basic.test_absolute_storage_dir_myowndir/" + base)
config_path = expected_path.encode("utf-8")
return self._storage_dir_test(
basedir,
config_path,
expected_path,
)
def __init__(self, client, accountfile):
self.client = client
self.passwords = BytesKeyDict()
pubkeys = BytesKeyDict()
self.rootcaps = BytesKeyDict()
with open(abspath_expanduser_unicode(accountfile), "rb") as f:
for line in f:
line = line.strip()
if line.startswith(b"#") or not line:
continue
name, passwd, rest = line.split(None, 2)
if passwd.startswith(b"ssh-"):
bits = rest.split()
keystring = b" ".join([passwd] + bits[:-1])
key = keys.Key.fromString(keystring)
rootcap = bits[-1]
pubkeys[name] = [key]
else:
self.passwords[name] = passwd
rootcap = rest
self.rootcaps[name] = rootcap
self._pubkeychecker = SSHPublicKeyChecker(InMemorySSHKeyDB(pubkeys))
def create_no_network_client(basedir):
"""
:return: a Deferred yielding an instance of _Client subclass which
does no actual networking but has the same API.
"""
basedir = abspath_expanduser_unicode(unicode(basedir))
fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
from allmydata.client import read_config
config = read_config(basedir, u'client.port')
storage_broker = NoNetworkStorageBroker()
client = _NoNetworkClient(
config,
main_tub=None,
i2p_provider=None,
tor_provider=None,
introducer_clients=[],
storage_farm_broker=storage_broker,
)
# this is a (pre-existing) reference-cycle and also a bad idea, see:
# https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2949
storage_broker.client = client
return defer.succeed(client)
def test_create_long_path(self):
"""
Even for paths with total length greater than 260 bytes,
``fileutil.abspath_expanduser_unicode`` produces a path on which other
path-related APIs can operate.
https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
documents certain Windows-specific path length limitations this test
is specifically intended to demonstrate can be overcome.
"""
workdir = u"test_create_long_path"
fileutil.make_dirs(workdir)
base_path = fileutil.abspath_expanduser_unicode(workdir)
base_length = len(base_path)
# Construct a path /just/ long enough to exercise the important case.
# It would be nice if we could just use a seemingly globally valid
# long file name (the `x...` portion) here - for example, a name 255
# bytes long- and a previous version of this test did just that.
# However, aufs imposes a 242 byte length limit on file names. Most
# other POSIX filesystems do allow names up to 255 bytes. It's not
# clear there's anything we can *do* about lower limits, though, and
# POSIX.1-2017 (and earlier) only requires that the maximum be at
# least 14 (!!!) bytes.
long_path = os.path.join(base_path, u'x' * (261 - base_length))
def _cleanup():
fileutil.remove(long_path)
self.addCleanup(_cleanup)
fileutil.write(long_path, b"test")
self.failUnless(os.path.exists(long_path))
self.failUnlessEqual(fileutil.read(long_path), b"test")
_cleanup()
self.failIf(os.path.exists(long_path))
def __init__(self, configparser, portnum_fname, basedir, config_fname):
"""
:param configparser: a ConfigParser instance
:param portnum_fname: filename to use for the port-number file
(a relative path inside basedir)
:param basedir: path to our "node directory", inside which all
configuration is managed
:param config_fname: the pathname actually used to create the
configparser (might be 'fake' if using in-memory data)
"""
self.portnum_fname = portnum_fname
self._basedir = abspath_expanduser_unicode(unicode(basedir))
self._config_fname = config_fname
self.config = configparser
nickname_utf8 = self.get_config("node", "nickname", "<unspecified>")
if isinstance(nickname_utf8, bytes): # Python 2
self.nickname = nickname_utf8.decode("utf-8")
else:
self.nickname = nickname_utf8
assert type(self.nickname) is unicode
def run(self):
options = self.options
nodeurl = options['node-url']
self.verbosity = 1
if options['quiet']:
self.verbosity = 0
if options['verbose']:
self.verbosity = 2
stdout = options.stdout
stderr = options.stderr
start_timestamp = datetime.datetime.now()
bdbfile = os.path.join(options["node-directory"], "private",
"backupdb.sqlite")
bdbfile = abspath_expanduser_unicode(bdbfile)
self.backupdb = backupdb.get_backupdb(bdbfile, stderr)
if not self.backupdb:
print("ERROR: Unable to load backup db.", file=stderr)
return 1
try:
rootcap, path = get_alias(options.aliases, options.to_dir,
DEFAULT_ALIAS)
except UnknownAliasError as e:
e.display(stderr)
return 1
to_url = nodeurl + "uri/%s/" % url_quote(rootcap)
if path:
to_url += escape_path(path)
if not to_url.endswith("/"):
to_url += "/"
archives_url = to_url + "Archives/"
archives_url = archives_url.rstrip("/")
to_url = to_url.rstrip("/")
# first step: make sure the target directory exists, as well as the
# Archives/ subdirectory.
resp = do_http("GET", archives_url + "?t=json")
if resp.status == 404:
resp = do_http("POST", archives_url + "?t=mkdir")
if resp.status != 200:
print(format_http_error("Unable to create target directory",
resp),
file=stderr)
return 1
# second step: process the tree
targets = list(
collect_backup_targets(
options.from_dir,
listdir_unicode,
self.options.filter_listdir,
))
completed = run_backup(
warn=self.warn,
upload_file=self.upload,
upload_directory=self.upload_directory,
targets=targets,
start_timestamp=start_timestamp,
stdout=stdout,
)
new_backup_dircap = completed.dircap
# third: attach the new backup to the list
now = time_format.iso_utc(int(time.time()), sep="_") + "Z"
put_child(archives_url, now, new_backup_dircap)
put_child(to_url, "Latest", new_backup_dircap)
print(completed.report(
self.verbosity,
self._files_checked,
self._directories_checked,
),
file=stdout)
# The command exits with code 2 if files or directories were skipped
if completed.any_skips():
return 2
# done!
return 0
def get_source_info(self, source_spec):
"""
This turns an argv string into a (Local|Tahoe)(File|Directory)Source.
"""
precondition(isinstance(source_spec, str), source_spec)
rootcap, path_utf8 = get_alias(self.aliases, source_spec, None)
path = path_utf8.decode("utf-8")
# any trailing slash is removed in abspath_expanduser_unicode(), so
# make a note of it here, to throw an error later
had_trailing_slash = path.endswith("/")
if rootcap == DefaultAliasMarker:
# no alias, so this is a local file
pathname = abspath_expanduser_unicode(path)
name = os.path.basename(pathname)
if not os.path.exists(pathname):
raise MissingSourceError(source_spec,
quotefn=quote_local_unicode_path)
if os.path.isdir(pathname):
t = LocalDirectorySource(self.progress, pathname, name)
else:
if had_trailing_slash:
raise FilenameWithTrailingSlashError(
source_spec, quotefn=quote_local_unicode_path)
if not os.path.isfile(pathname):
raise WeirdSourceError(pathname)
t = LocalFileSource(pathname, name) # non-empty
else:
# this is a tahoe object
url = self.nodeurl + "uri/%s" % url_quote(rootcap)
name = None
if path:
if path.endswith("/"):
path = path[:-1]
url += "/" + escape_path(path)
last_slash = path.rfind(u"/")
name = path
if last_slash != -1:
name = path[last_slash + 1:]
resp = do_http("GET", url + "?t=json")
if resp.status == 404:
raise MissingSourceError(source_spec)
elif resp.status != 200:
raise HTTPError(
"Error examining source %s" % quote_output(source_spec),
resp)
parsed = json.loads(resp.read())
nodetype, d = parsed
if nodetype == "dirnode":
t = TahoeDirectorySource(self.nodeurl, self.cache,
self.progress, name)
t.init_from_parsed(parsed)
else:
if had_trailing_slash:
raise FilenameWithTrailingSlashError(source_spec)
writecap = to_bytes(d.get("rw_uri"))
readcap = to_bytes(d.get("ro_uri"))
mutable = d.get("mutable",
False) # older nodes don't provide it
t = TahoeFileSource(self.nodeurl, mutable, writecap, readcap,
name)
return t
import os, sys, urllib
import codecs
from twisted.python import usage
from allmydata.util.assertutil import precondition
from allmydata.util.encodingutil import unicode_to_url, quote_output, argv_to_abspath
from allmydata.util.fileutil import abspath_expanduser_unicode
_default_nodedir = None
if sys.platform == 'win32':
from allmydata.windows import registry
path = registry.get_base_dir_path()
if path:
precondition(isinstance(path, unicode), path)
_default_nodedir = abspath_expanduser_unicode(path)
if _default_nodedir is None:
path = abspath_expanduser_unicode(u"~/.tahoe")
precondition(isinstance(path, unicode), path)
_default_nodedir = path
def get_default_nodedir():
return _default_nodedir
class BaseOptions(usage.Options):
# unit tests can override these to point at StringIO instances
stdin = sys.stdin
stdout = sys.stdout
stderr = sys.stderr
def test_join_failures(self):
self.basedir = "cli/MagicFolder/create-join-failures"
os.makedirs(self.basedir)
self.set_up_grid(oneshare=True)
local_dir = os.path.join(self.basedir, "magic")
os.mkdir(local_dir)
abs_local_dir_u = abspath_expanduser_unicode(unicode(local_dir),
long_path=False)
self.invite_code = None
d = self.do_create_magic_folder(0)
d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname))
def get_invite_code_and_join((rc, stdout, stderr)):
self.failUnlessEqual(rc, 0)
self.invite_code = stdout.strip()
return self.do_join(0, unicode(local_dir), self.invite_code)
d.addCallback(get_invite_code_and_join)
def get_caps(ign):
self.collective_dircap, self.upload_dircap = self.get_caps_from_files(
0)
d.addCallback(get_caps)
d.addCallback(
lambda ign: self.check_joined_config(0, self.upload_dircap))
d.addCallback(lambda ign: self.check_config(0, abs_local_dir_u))
def check_success(result):
(rc, out, err) = result
self.failUnlessEqual(rc, 0, out + err)
def check_failure(result):
(rc, out, err) = result
self.failIfEqual(rc, 0)
def leave(ign):
return self.do_cli("magic-folder", "leave", client_num=0)
d.addCallback(leave)
d.addCallback(check_success)
magic_folder_db_file = os.path.join(self.get_clientdir(i=0),
u"private",
u"magicfolder_default.sqlite")
def check_join_if_file(my_file):
fileutil.write(my_file, "my file data")
d2 = self.do_cli("magic-folder",
"join",
self.invite_code,
local_dir,
client_num=0)
d2.addCallback(check_failure)
return d2
for my_file in [magic_folder_db_file]:
d.addCallback(lambda ign, my_file: check_join_if_file(my_file),
my_file)
d.addCallback(leave)
# we didn't successfully join, so leaving should be an error
d.addCallback(check_failure)
return d
def argv_to_abspath(s):
"""
Convenience function to decode an argv element to an absolute path, with ~ expanded.
If this fails, raise a UsageError.
"""
return abspath_expanduser_unicode(argv_to_unicode(s))
def setup_alice_and_bob(self, alice_clock=reactor, bob_clock=reactor):
self.set_up_grid(num_clients=2, oneshare=True)
self.alice_magicfolder = None
self.bob_magicfolder = None
alice_magic_dir = abspath_expanduser_unicode(u"Alice-magic",
base=self.basedir)
self.mkdir_nonascii(alice_magic_dir)
bob_magic_dir = abspath_expanduser_unicode(u"Bob-magic",
base=self.basedir)
self.mkdir_nonascii(bob_magic_dir)
# Alice creates a Magic Folder, invites herself and joins.
d = self.do_create_magic_folder(0)
d.addCallback(lambda ign: self.do_invite(0, self.alice_nickname))
def get_invite_code(result):
self.invite_code = result[1].strip()
d.addCallback(get_invite_code)
d.addCallback(
lambda ign: self.do_join(0, alice_magic_dir, self.invite_code))
def get_alice_caps(ign):
self.alice_collective_dircap, self.alice_upload_dircap = self.get_caps_from_files(
0)
d.addCallback(get_alice_caps)
d.addCallback(
lambda ign: self.check_joined_config(0, self.alice_upload_dircap))
d.addCallback(lambda ign: self.check_config(0, alice_magic_dir))
def get_Alice_magicfolder(result):
self.alice_magicfolder = self.init_magicfolder(
0, self.alice_upload_dircap, self.alice_collective_dircap,
alice_magic_dir, alice_clock)
return result
d.addCallback(get_Alice_magicfolder)
# Alice invites Bob. Bob joins.
d.addCallback(lambda ign: self.do_invite(0, self.bob_nickname))
def get_invite_code(result):
self.invite_code = result[1].strip()
d.addCallback(get_invite_code)
d.addCallback(
lambda ign: self.do_join(1, bob_magic_dir, self.invite_code))
def get_bob_caps(ign):
self.bob_collective_dircap, self.bob_upload_dircap = self.get_caps_from_files(
1)
d.addCallback(get_bob_caps)
d.addCallback(
lambda ign: self.check_joined_config(1, self.bob_upload_dircap))
d.addCallback(lambda ign: self.check_config(1, bob_magic_dir))
def get_Bob_magicfolder(result):
self.bob_magicfolder = self.init_magicfolder(
1, self.bob_upload_dircap, self.bob_collective_dircap,
bob_magic_dir, bob_clock)
return result
d.addCallback(get_Bob_magicfolder)
return d
def ensure_text_and_abspath_expanduser_unicode(basedir):
# type: (Union[bytes, str]) -> str
return abspath_expanduser_unicode(ensure_text(basedir))
def setUp(self):
self.account_file = filepath.FilePath(self.mktemp())
self.account_file.setContent(DUMMY_ACCOUNTS)
abspath = abspath_expanduser_unicode(str(self.account_file.path))
self.checker = auth.AccountFileChecker(None, abspath)
def __init__(self, client, accountfile):
self.client = client
path = abspath_expanduser_unicode(accountfile)
with open_account_file(path) as f:
self.rootcaps, pubkeys = load_account_file(f)
self._pubkeychecker = SSHPublicKeyChecker(InMemorySSHKeyDB(pubkeys))
def test_abspath_expanduser_unicode(self):
self.failUnlessRaises(AssertionError,
fileutil.abspath_expanduser_unicode,
b"bytestring")
saved_cwd = os.path.normpath(os.getcwd())
if PY2:
saved_cwd = saved_cwd.decode("utf8")
abspath_cwd = fileutil.abspath_expanduser_unicode(u".")
abspath_cwd_notlong = fileutil.abspath_expanduser_unicode(
u".", long_path=False)
self.failUnless(isinstance(saved_cwd, str), saved_cwd)
self.failUnless(isinstance(abspath_cwd, str), abspath_cwd)
if sys.platform == "win32":
self.failUnlessReallyEqual(
abspath_cwd, fileutil.to_windows_long_path(saved_cwd))
else:
self.failUnlessReallyEqual(abspath_cwd, saved_cwd)
self.failUnlessReallyEqual(abspath_cwd_notlong, saved_cwd)
self.failUnlessReallyEqual(
fileutil.to_windows_long_path(u"\\\\?\\foo"), u"\\\\?\\foo")
self.failUnlessReallyEqual(
fileutil.to_windows_long_path(u"\\\\.\\foo"), u"\\\\.\\foo")
self.failUnlessReallyEqual(
fileutil.to_windows_long_path(u"\\\\server\\foo"),
u"\\\\?\\UNC\\server\\foo")
self.failUnlessReallyEqual(fileutil.to_windows_long_path(u"C:\\foo"),
u"\\\\?\\C:\\foo")
self.failUnlessReallyEqual(
fileutil.to_windows_long_path(u"C:\\foo/bar"),
u"\\\\?\\C:\\foo\\bar")
# adapted from <http://svn.python.org/view/python/branches/release26-maint/Lib/test/test_posixpath.py?view=markup&pathrev=78279#test_abspath>
foo = fileutil.abspath_expanduser_unicode(u"foo")
self.failUnless(foo.endswith(u"%sfoo" % (os.path.sep, )), foo)
foobar = fileutil.abspath_expanduser_unicode(u"bar", base=foo)
self.failUnless(
foobar.endswith(u"%sfoo%sbar" % (os.path.sep, os.path.sep)),
foobar)
if sys.platform == "win32":
# This is checking that a drive letter is added for a path without one.
baz = fileutil.abspath_expanduser_unicode(u"\\baz")
self.failUnless(baz.startswith(u"\\\\?\\"), baz)
self.failUnlessReallyEqual(baz[5:], u":\\baz")
bar = fileutil.abspath_expanduser_unicode(u"\\bar", base=baz)
self.failUnless(bar.startswith(u"\\\\?\\"), bar)
self.failUnlessReallyEqual(bar[5:], u":\\bar")
# not u":\\baz\\bar", because \bar is absolute on the current drive.
self.failUnlessReallyEqual(baz[4], bar[4]) # same drive
baz_notlong = fileutil.abspath_expanduser_unicode(u"\\baz",
long_path=False)
self.failIf(baz_notlong.startswith(u"\\\\?\\"), baz_notlong)
self.failUnlessReallyEqual(baz_notlong[1:], u":\\baz")
bar_notlong = fileutil.abspath_expanduser_unicode(u"\\bar",
base=baz_notlong,
long_path=False)
self.failIf(bar_notlong.startswith(u"\\\\?\\"), bar_notlong)
self.failUnlessReallyEqual(bar_notlong[1:], u":\\bar")
# not u":\\baz\\bar", because \bar is absolute on the current drive.
self.failUnlessReallyEqual(baz_notlong[0],
bar_notlong[0]) # same drive
self.failIfIn(u"~", fileutil.abspath_expanduser_unicode(u"~"))
self.failIfIn(
u"~", fileutil.abspath_expanduser_unicode(u"~", long_path=False))
cwds = ['cwd']
try:
cwds.append(u'\xe7w\xf0'.encode(sys.getfilesystemencoding()
or 'ascii'))
except UnicodeEncodeError:
pass # the cwd can't be encoded -- test with ascii cwd only
for cwd in cwds:
try:
os.mkdir(cwd)
os.chdir(cwd)
for upath in (u'', u'fuu', u'f\xf9\xf9', u'/fuu', u'U:\\',
u'~'):
uabspath = fileutil.abspath_expanduser_unicode(upath)
self.failUnless(isinstance(uabspath, str), uabspath)
uabspath_notlong = fileutil.abspath_expanduser_unicode(
upath, long_path=False)
self.failUnless(isinstance(uabspath_notlong, str),
uabspath_notlong)
finally:
os.chdir(saved_cwd)
def call_file(name, *args):
ns.called = True
self.failUnlessEqual(name,
abspath_expanduser_unicode(exclude_file))
return StringIO()
class _Config(object):
"""
Manages configuration of a Tahoe 'node directory'.
Note: all this code and functionality was formerly in the Node
class; names and funtionality have been kept the same while moving
the code. It probably makes sense for several of these APIs to
have better names.
:ivar ConfigParser config: The actual configuration values.
:ivar str portnum_fname: filename to use for the port-number file (a
relative path inside basedir).
:ivar str _basedir: path to our "node directory", inside which all
configuration is managed.
:ivar (FilePath|NoneType) config_path: The path actually used to create
the configparser (might be ``None`` if using in-memory data).
:ivar ValidConfiguration valid_config_sections: The validator for the
values in this configuration.
"""
config = attr.ib(
validator=attr.validators.instance_of(configparser.ConfigParser))
portnum_fname = attr.ib()
_basedir = attr.ib(converter=lambda basedir: abspath_expanduser_unicode(
ensure_text(basedir)), )
config_path = attr.ib(validator=attr.validators.optional(
attr.validators.instance_of(FilePath), ), )
valid_config_sections = attr.ib(
default=configutil.ValidConfiguration.everything(),
validator=attr.validators.instance_of(configutil.ValidConfiguration),
)
@property
def nickname(self):
nickname = self.get_config("node", "nickname", u"<unspecified>")
assert isinstance(nickname, str)
return nickname
@property
def _config_fname(self):
if self.config_path is None:
return "<string>"
return self.config_path.path
def write_config_file(self, name, value, mode="w"):
"""
writes the given 'value' into a file called 'name' in the config
directory
"""
fn = os.path.join(self._basedir, name)
try:
fileutil.write(fn, value, mode)
except EnvironmentError:
log.err(
Failure(),
"Unable to write config file '{}'".format(fn),
)
def items(self, section, default=_None):
try:
return self.config.items(section)
except configparser.NoSectionError:
if default is _None:
raise
return default
def get_config(self, section, option, default=_None, boolean=False):
try:
if boolean:
return self.config.getboolean(section, option)
item = self.config.get(section, option)
if option.endswith(".furl") and '#' in item:
raise UnescapedHashError(section, option, item)
return item
except (configparser.NoOptionError, configparser.NoSectionError):
if default is _None:
raise MissingConfigEntry(
"{} is missing the [{}]{} entry".format(
quote_output(self._config_fname),
section,
option,
))
return default
def set_config(self, section, option, value):
"""
Set a config option in a section and re-write the tahoe.cfg file
:param str section: The name of the section in which to set the
option.
:param str option: The name of the option to set.
:param str value: The value of the option.
:raise UnescapedHashError: If the option holds a fURL and there is a
``#`` in the value.
"""
if option.endswith(".furl") and "#" in value:
raise UnescapedHashError(section, option, value)
copied_config = configutil.copy_config(self.config)
configutil.set_config(copied_config, section, option, value)
configutil.validate_config(
self._config_fname,
copied_config,
self.valid_config_sections,
)
if self.config_path is not None:
configutil.write_config(self.config_path, copied_config)
self.config = copied_config
def get_config_from_file(self, name, required=False):
"""Get the (string) contents of a config file, or None if the file
did not exist. If required=True, raise an exception rather than
returning None. Any leading or trailing whitespace will be stripped
from the data."""
fn = os.path.join(self._basedir, name)
try:
return fileutil.read(fn).strip()
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise # we only care about "file doesn't exist"
if not required:
return None
raise
def get_or_create_private_config(self, name, default=_None):
"""Try to get the (string) contents of a private config file (which
is a config file that resides within the subdirectory named
'private'), and return it. Any leading or trailing whitespace will be
stripped from the data.
If the file does not exist, and default is not given, report an error.
If the file does not exist and a default is specified, try to create
it using that default, and then return the value that was written.
If 'default' is a string, use it as a default value. If not, treat it
as a zero-argument callable that is expected to return a string.
"""
privname = os.path.join(self._basedir, "private", name)
try:
value = fileutil.read(privname, mode="r")
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise # we only care about "file doesn't exist"
if default is _None:
raise MissingConfigEntry(
"The required configuration file %s is missing." %
(quote_output(privname), ))
if isinstance(default, bytes):
default = str(default, "utf-8")
if isinstance(default, str):
value = default
else:
value = default()
fileutil.write(privname, value)
return value.strip()
def write_private_config(self, name, value):
"""Write the (string) contents of a private config file (which is a
config file that resides within the subdirectory named 'private'), and
return it.
"""
if isinstance(value, str):
value = value.encode("utf-8")
privname = os.path.join(self._basedir, "private", name)
with open(privname, "wb") as f:
f.write(value)
def get_private_config(self, name, default=_None):
"""Read the (native string) contents of a private config file (a
config file that resides within the subdirectory named 'private'),
and return it. Return a default, or raise an error if one was not
given.
"""
privname = os.path.join(self._basedir, "private", name)
try:
return fileutil.read(privname, mode="r").strip()
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise # we only care about "file doesn't exist"
if default is _None:
raise MissingConfigEntry(
"The required configuration file %s is missing." %
(quote_output(privname), ))
return default
def get_private_path(self, *args):
"""
returns an absolute path inside the 'private' directory with any
extra args join()-ed
"""
return os.path.join(self._basedir, "private", *args)
def get_config_path(self, *args):
"""
returns an absolute path inside the config directory with any
extra args join()-ed
"""
# note: we re-expand here (_basedir already went through this
# expanduser function) in case the path we're being asked for
# has embedded ".."'s in it
return abspath_expanduser_unicode(os.path.join(self._basedir, *args))
def get_introducer_configuration(self):
"""
Get configuration for introducers.
:return {unicode: (unicode, FilePath)}: A mapping from introducer
petname to a tuple of the introducer's fURL and local cache path.
"""
introducers_yaml_filename = self.get_private_path("introducers.yaml")
introducers_filepath = FilePath(introducers_yaml_filename)
def get_cache_filepath(petname):
return FilePath(
self.get_private_path(
"introducer_{}_cache.yaml".format(petname)), )
try:
with introducers_filepath.open() as f:
introducers_yaml = safe_load(f)
if introducers_yaml is None:
raise EnvironmentError(
EPERM,
"Can't read '{}'".format(introducers_yaml_filename),
introducers_yaml_filename,
)
introducers = {
petname: config["furl"]
for petname, config in introducers_yaml.get(
"introducers", {}).items()
}
non_strs = list(k for k in introducers.keys()
if not isinstance(k, str))
if non_strs:
raise TypeError(
"Introducer petnames {!r} should have been str".format(
non_strs, ), )
non_strs = list(v for v in introducers.values()
if not isinstance(v, str))
if non_strs:
raise TypeError(
"Introducer fURLs {!r} should have been str".format(
non_strs, ), )
log.msg("found {} introducers in {!r}".format(
len(introducers),
introducers_yaml_filename,
))
except EnvironmentError as e:
if e.errno != ENOENT:
raise
introducers = {}
# supported the deprecated [client]introducer.furl item in tahoe.cfg
tahoe_cfg_introducer_furl = self.get_config("client",
"introducer.furl", None)
if tahoe_cfg_introducer_furl == "None":
raise ValueError("tahoe.cfg has invalid 'introducer.furl = None':"
" to disable it omit the key entirely")
if tahoe_cfg_introducer_furl:
warn(
"tahoe.cfg [client]introducer.furl is deprecated; "
"use private/introducers.yaml instead.",
category=DeprecationWarning,
stacklevel=-1,
)
if "default" in introducers:
raise ValueError(
"'default' introducer furl cannot be specified in tahoe.cfg and introducers.yaml;"
" please fix impossible configuration.")
introducers['default'] = tahoe_cfg_introducer_furl
return {
petname: (furl, get_cache_filepath(petname))
for (petname, furl) in introducers.items()
}
def check_file(self, path, use_timestamps=True):
"""I will tell you if a given local file needs to be uploaded or not,
by looking in a database and seeing if I have a record of this file
having been uploaded earlier.
I return a FileResults object, synchronously. If r.was_uploaded()
returns False, you should upload the file. When you are finished
uploading it, call r.did_upload(filecap), so I can update my
database.
If was_uploaded() returns a filecap, you might be able to avoid an
upload. Call r.should_check(), and if it says False, you can skip the
upload and use the filecap returned by was_uploaded().
If should_check() returns True, you should perform a filecheck on the
filecap returned by was_uploaded(). If the check indicates the file
is healthy, please call r.did_check_healthy(checker_results) so I can
update the database, using the de-JSONized response from the webapi
t=check call for 'checker_results'. If the check indicates the file
is not healthy, please upload the file and call r.did_upload(filecap)
when you're done.
I use_timestamps=True (the default), I will compare ctime and mtime
of the local file against an entry in my database, and consider the
file to be unchanged if ctime, mtime, and filesize are all the same
as the earlier version. If use_timestamps=False, I will not trust the
timestamps, so more files (perhaps all) will be marked as needing
upload. A future version of this database may hash the file to make
equality decisions, in which case use_timestamps=False will not
always imply r.must_upload()==True.
'path' points to a local file on disk, possibly relative to the
current working directory. The database stores absolute pathnames.
"""
path = abspath_expanduser_unicode(path)
s = os.stat(path)
size = s[stat.ST_SIZE]
ctime = s[stat.ST_CTIME]
mtime = s[stat.ST_MTIME]
now = time.time()
c = self.cursor
c.execute(
"SELECT size,mtime,ctime,fileid"
" FROM local_files"
" WHERE path=?", (path, ))
row = self.cursor.fetchone()
if not row:
return FileResult(self, None, False, path, mtime, ctime, size)
(last_size, last_mtime, last_ctime, last_fileid) = row
c.execute(
"SELECT caps.filecap, last_upload.last_checked"
" FROM caps,last_upload"
" WHERE caps.fileid=? AND last_upload.fileid=?",
(last_fileid, last_fileid))
row2 = c.fetchone()
if ((last_size != size or not use_timestamps or last_mtime != mtime
or last_ctime != ctime) # the file has been changed
or
(not row2) # we somehow forgot where we put the file last time
):
c.execute("DELETE FROM local_files WHERE path=?", (path, ))
self.connection.commit()
return FileResult(self, None, False, path, mtime, ctime, size)
# at this point, we're allowed to assume the file hasn't been changed
(filecap, last_checked) = row2
age = now - last_checked
probability = ((age - self.NO_CHECK_BEFORE) /
(self.ALWAYS_CHECK_AFTER - self.NO_CHECK_BEFORE))
probability = min(max(probability, 0.0), 1.0)
should_check = bool(random.random() < probability)
return FileResult(self, to_str(filecap), should_check, path, mtime,
ctime, size)
