def catalog_shares(options):
from allmydata.util.encodingutil import listdir_unicode, quote_output
out = options.stdout
err = options.stderr
now = time.time()
for d in options.nodedirs:
d = os.path.join(d, "storage/shares")
try:
abbrevs = listdir_unicode(d)
except EnvironmentError:
# ignore nodes that have storage turned off altogether
pass
else:
for abbrevdir in sorted(abbrevs):
if abbrevdir == "incoming":
continue
abbrevdir = os.path.join(d, abbrevdir)
# this tool may get run against bad disks, so we can't assume
# that listdir_unicode will always succeed. Try to catalog as much
# as possible.
try:
sharedirs = listdir_unicode(abbrevdir)
for si_s in sorted(sharedirs):
si_dir = os.path.join(abbrevdir, si_s)
catalog_shares_one_abbrevdir(si_s, si_dir, now, out,err)
except:
print >>err, "Error processing %s" % quote_output(abbrevdir)
failure.Failure().printTraceback(err)
return 0
def catalog_shares(options):
from allmydata.util.encodingutil import listdir_unicode, quote_output
out = options.stdout
err = options.stderr
now = time.time()
for d in options.nodedirs:
d = os.path.join(d, "storage", "shares")
try:
abbrevs = listdir_unicode(d)
except EnvironmentError:
# ignore nodes that have storage turned off altogether
pass
else:
for abbrevdir in sorted(abbrevs):
if abbrevdir == "incoming":
continue
abbrevdir = os.path.join(d, abbrevdir)
# this tool may get run against bad disks, so we can't assume
# that listdir_unicode will always succeed. Try to catalog as much
# as possible.
try:
sharedirs = listdir_unicode(abbrevdir)
for si_s in sorted(sharedirs):
si_dir = os.path.join(abbrevdir, si_s)
catalog_shares_one_abbrevdir(si_s, si_dir, now, out,
err)
except:
print("Error processing %s" % quote_output(abbrevdir),
file=err)
failure.Failure().printTraceback(err)
return 0
def process(self, localpath):
precondition_abspath(localpath)
# returns newdircap
quoted_path = quote_local_unicode_path(localpath)
self.verboseprint("processing %s" % (quoted_path,))
create_contents = {} # childname -> (type, rocap, metadata)
compare_contents = {} # childname -> rocap
try:
children = listdir_unicode(localpath)
except EnvironmentError:
self.directories_skipped += 1
self.warn("WARNING: permission denied on directory %s" % (quoted_path,))
children = []
except FilenameEncodingError:
self.directories_skipped += 1
self.warn("WARNING: could not list directory %s due to a filename encoding error" % (quoted_path,))
children = []
for child in self.options.filter_listdir(children):
assert isinstance(child, unicode), child
childpath = os.path.join(localpath, child)
# note: symlinks to directories are both islink() and isdir()
if os.path.isdir(childpath) and not os.path.islink(childpath):
metadata = get_local_metadata(childpath)
# recurse on the child directory
childcap = self.process(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("dirnode", childcap, metadata)
compare_contents[child] = childcap
elif os.path.isfile(childpath) and not os.path.islink(childpath):
try:
childcap, metadata = self.upload(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("filenode", childcap, metadata)
compare_contents[child] = childcap
except EnvironmentError:
self.files_skipped += 1
self.warn("WARNING: permission denied on file %s" % quote_local_unicode_path(childpath))
else:
self.files_skipped += 1
if os.path.islink(childpath):
self.warn("WARNING: cannot backup symlink %s" % quote_local_unicode_path(childpath))
else:
self.warn("WARNING: cannot backup special file %s" % quote_local_unicode_path(childpath))
must_create, r = self.check_backupdb_directory(compare_contents)
if must_create:
self.verboseprint(" creating directory for %s" % quote_local_unicode_path(localpath))
newdircap = mkdir(create_contents, self.options)
assert isinstance(newdircap, str)
if r:
r.did_create(newdircap)
self.directories_created += 1
return newdircap
else:
self.verboseprint(" re-using old directory for %s" % quote_local_unicode_path(localpath))
self.directories_reused += 1
return r.was_created()
def create_node(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >> err, "The base directory %s is not empty." % quote_local_unicode_path(
basedir)
print >> err, "To avoid clobbering anything, I am going to quit now."
print >> err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
write_client_config(c, config)
print >> out, "Node created in %s" % quote_local_unicode_path(basedir)
tahoe_cfg = quote_local_unicode_path(os.path.join(basedir, "tahoe.cfg"))
if not config.get("introducer", ""):
print >> out, " Please set [client]introducer.furl= in %s!" % tahoe_cfg
print >> out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >> out, " Please set [node]nickname= in %s" % tahoe_cfg
defer.returnValue(0)
def test_unicode(self):
skip_if_cannot_represent_filename(u"f\u00f6\u00f6.txt")
skip_if_cannot_represent_filename(u"b\u00e5r.txt")
self.basedir = basedir = os.path.join("backupdb", "unicode")
fileutil.make_dirs(basedir)
dbfile = os.path.join(basedir, "dbfile")
bdb = self.create(dbfile)
self.writeto(u"f\u00f6\u00f6.txt", "foo.txt")
files = [fn for fn in listdir_unicode(unicode(basedir)) if fn.endswith(".txt")]
self.failUnlessEqual(len(files), 1)
foo_fn = os.path.join(basedir, files[0])
#print foo_fn, type(foo_fn)
r = bdb.check_file(foo_fn)
self.failUnlessEqual(r.was_uploaded(), False)
r.did_upload("foo-cap")
r = bdb.check_file(foo_fn)
self.failUnlessEqual(r.was_uploaded(), "foo-cap")
self.failUnlessEqual(r.should_check(), False)
bar_fn = self.writeto(u"b\u00e5r.txt", "bar.txt")
#print bar_fn, type(bar_fn)
r = bdb.check_file(bar_fn)
self.failUnlessEqual(r.was_uploaded(), False)
r.did_upload("bar-cap")
r = bdb.check_file(bar_fn)
self.failUnlessEqual(r.was_uploaded(), "bar-cap")
self.failUnlessEqual(r.should_check(), False)
def create_introducer(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >> err, "The base directory %s is not empty." % quote_output(
basedir)
print >> err, "To avoid clobbering anything, I am going to quit now."
print >> err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
f = open(os.path.join(basedir, "tahoe-introducer.tac"), "w")
f.write(introducer_tac)
f.close()
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.close()
print >> out, "Introducer created in %s" % quote_output(basedir)
return 0
def create_node(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
write_client_config(c, config)
print >>out, "Node created in %s" % quote_local_unicode_path(basedir)
if not config.get("introducer", ""):
print >>out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >>out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >>out, " Please set [node]nickname= in tahoe.cfg"
defer.returnValue(0)
def create_stats_gatherer(config):
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print("The base directory %s is not empty." % quote_output(basedir), file=err)
print("To avoid clobbering anything, I am going to quit now.", file=err)
print("Please use a different directory, or empty this one.", file=err)
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "stats-gatherer")
if config["hostname"]:
portnum = iputil.allocate_tcp_port()
location = "tcp:%s:%d" % (config["hostname"], portnum)
port = "tcp:%d" % portnum
else:
location = config["location"]
port = config["port"]
fileutil.write(os.path.join(basedir, "location"), location+"\n")
fileutil.write(os.path.join(basedir, "port"), port+"\n")
return 0
def process(self, localpath):
precondition_abspath(localpath)
# returns newdircap
quoted_path = quote_local_unicode_path(localpath)
self.verboseprint("processing %s" % (quoted_path,))
create_contents = {} # childname -> (type, rocap, metadata)
compare_contents = {} # childname -> rocap
try:
children = listdir_unicode(localpath)
except EnvironmentError:
self.directories_skipped += 1
self.warn("WARNING: permission denied on directory %s" % (quoted_path,))
children = []
except FilenameEncodingError:
self.directories_skipped += 1
self.warn("WARNING: could not list directory %s due to a filename encoding error" % (quoted_path,))
children = []
for child in self.options.filter_listdir(children):
assert isinstance(child, unicode), child
childpath = os.path.join(localpath, child)
# note: symlinks to directories are both islink() and isdir()
if os.path.isdir(childpath) and not os.path.islink(childpath):
metadata = get_local_metadata(childpath)
# recurse on the child directory
childcap = self.process(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("dirnode", childcap, metadata)
compare_contents[child] = childcap
elif os.path.isfile(childpath) and not os.path.islink(childpath):
try:
childcap, metadata = self.upload(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("filenode", childcap, metadata)
compare_contents[child] = childcap
except EnvironmentError:
self.files_skipped += 1
self.warn("WARNING: permission denied on file %s" % quote_local_unicode_path(childpath))
else:
self.files_skipped += 1
if os.path.islink(childpath):
self.warn("WARNING: cannot backup symlink %s" % quote_local_unicode_path(childpath))
else:
self.warn("WARNING: cannot backup special file %s" % quote_local_unicode_path(childpath))
must_create, r = self.check_backupdb_directory(compare_contents)
if must_create:
self.verboseprint(" creating directory for %s" % quote_local_unicode_path(localpath))
newdircap = mkdir(create_contents, self.options)
assert isinstance(newdircap, str)
if r:
r.did_create(newdircap)
self.directories_created += 1
return newdircap
else:
self.verboseprint(" re-using old directory for %s" % quote_local_unicode_path(localpath))
self.directories_reused += 1
return r.was_created()
def create_stats_gatherer(config):
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print("The base directory %s is not empty." %
quote_output(basedir),
file=err)
print("To avoid clobbering anything, I am going to quit now.",
file=err)
print("Please use a different directory, or empty this one.",
file=err)
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "stats-gatherer")
if config["hostname"]:
portnum = iputil.allocate_tcp_port()
location = "tcp:%s:%d" % (config["hostname"], portnum)
port = "tcp:%d" % portnum
else:
location = config["location"]
port = config["port"]
fileutil.write(os.path.join(basedir, "location"), location + "\n")
fileutil.write(os.path.join(basedir, "port"), port + "\n")
return 0
def test_unicode(self):
skip_if_cannot_represent_filename(u"f\u00f6\u00f6.txt")
skip_if_cannot_represent_filename(u"b\u00e5r.txt")
self.basedir = basedir = os.path.join("backupdb", "unicode")
fileutil.make_dirs(basedir)
dbfile = os.path.join(basedir, "dbfile")
bdb = self.create(dbfile)
self.writeto(u"f\u00f6\u00f6.txt", "foo.txt")
files = [
fn for fn in listdir_unicode(str(basedir)) if fn.endswith(".txt")
]
self.failUnlessEqual(len(files), 1)
foo_fn = os.path.join(basedir, files[0])
#print(foo_fn, type(foo_fn))
r = bdb.check_file(foo_fn)
self.failUnlessEqual(r.was_uploaded(), False)
r.did_upload(b"foo-cap")
r = bdb.check_file(foo_fn)
self.failUnlessEqual(r.was_uploaded(), b"foo-cap")
self.failUnlessEqual(r.should_check(), False)
bar_fn = self.writeto(u"b\u00e5r.txt", "bar.txt")
#print(bar_fn, type(bar_fn))
r = bdb.check_file(bar_fn)
self.failUnlessEqual(r.was_uploaded(), False)
r.did_upload(b"bar-cap")
r = bdb.check_file(bar_fn)
self.failUnlessEqual(r.was_uploaded(), b"bar-cap")
self.failUnlessEqual(r.should_check(), False)
def test_listdir_unicode(self):
if 'dirlist' not in dir(self):
return
try:
u"test".encode(self.filesystem_encoding)
except (LookupError, AttributeError):
raise unittest.SkipTest(
"This platform does not support the '%s' filesystem encoding "
"that we are testing for the benefit of a different platform."
% (self.filesystem_encoding, ))
def call_os_listdir(path):
return self.dirlist
self.patch(os, 'listdir', call_os_listdir)
def call_sys_getfilesystemencoding():
return self.filesystem_encoding
self.patch(sys, 'getfilesystemencoding',
call_sys_getfilesystemencoding)
_reload()
filenames = listdir_unicode(u'/dummy')
self.failUnlessEqual(set([normalize(fname) for fname in filenames]),
set(TEST_FILENAMES))
def test_mkdir_open_exists_abspath_listdir_expanduser(self):
skip_if_cannot_represent_filename(lumiere_nfc)
try:
os.mkdir(lumiere_nfc)
except EnvironmentError as e:
raise unittest.SkipTest("%r\nIt is possible that the filesystem on which this test is being run "
"does not support Unicode, even though the platform does." % (e,))
fn = lumiere_nfc + u'/' + lumiere_nfc + u'.txt'
open(fn, 'wb').close()
self.failUnless(os.path.exists(fn))
if PY2:
getcwdu = os.getcwdu
else:
getcwdu = os.getcwd
self.failUnless(os.path.exists(os.path.join(getcwdu(), fn)))
filenames = listdir_unicode(lumiere_nfc)
# We only require that the listing includes a filename that is canonically equivalent
# to lumiere_nfc (on Mac OS X, it will be the NFD equivalent).
self.failUnlessIn(lumiere_nfc + u".txt", set([encodingutil.normalize(fname) for fname in filenames]))
expanded = fileutil.expanduser(u"~/" + lumiere_nfc)
self.failIfIn(u"~", expanded)
self.failUnless(expanded.endswith(lumiere_nfc), expanded)
def create_introducer(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >> err, "The base directory %s is not empty." % quote_local_unicode_path(
basedir)
print >> err, "To avoid clobbering anything, I am going to quit now."
print >> err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "introducer")
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
print >> out, "Introducer created in %s" % quote_local_unicode_path(
basedir)
defer.returnValue(0)
class StdlibUnicode(unittest.TestCase):
"""This mainly tests that some of the stdlib functions support Unicode paths, but also that
listdir_unicode works for valid filenames."""
def skip_if_cannot_represent_filename(self, u):
enc = get_filesystem_encoding()
if not unicode_platform():
try:
u.encode(enc)
except UnicodeEncodeError:
raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
def test_mkdir_open_exists_abspath_listdir_expanduser(self):
self.skip_if_cannot_represent_filename(lumiere_nfc)
try:
os.mkdir(lumiere_nfc)
except EnvironmentError, e:
raise unittest.SkipTest("%r\nIt is possible that the filesystem on which this test is being run "
"does not support Unicode, even though the platform does." % (e,))
fn = lumiere_nfc + u'/' + lumiere_nfc + u'.txt'
open(fn, 'wb').close()
self.failUnless(os.path.exists(fn))
self.failUnless(os.path.exists(os.path.join(os.getcwdu(), fn)))
filenames = listdir_unicode(lumiere_nfc)
# We only require that the listing includes a filename that is canonically equivalent
# to lumiere_nfc (on Mac OS X, it will be the NFD equivalent).
self.failUnlessIn(lumiere_nfc + ".txt", set([normalize(fname) for fname in filenames]))
expanded = os.path.expanduser("~/" + lumiere_nfc)
self.failIfIn("~", expanded)
self.failUnless(expanded.endswith(lumiere_nfc), expanded)
def start(opts, out=sys.stdout, err=sys.stderr):
basedir = opts['basedir']
print >>out, "STARTING", quote_output(basedir)
if not os.path.isdir(basedir):
print >>err, "%s does not look like a directory at all" % quote_output(basedir)
return 1
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = str(fn)
break
else:
print >>err, "%s does not look like a node directory (no .tac file)" % quote_output(basedir)
return 1
if "client" in tac:
nodetype = "client"
elif "introducer" in tac:
nodetype = "introducer"
else:
nodetype = "unknown (%s)" % tac
args = ["twistd", "-y", tac]
if opts["syslog"]:
args.append("--syslog")
elif nodetype in ("client", "introducer"):
fileutil.make_dirs(os.path.join(basedir, "logs"))
args.extend(["--logfile", os.path.join("logs", "twistd.log")])
if opts["profile"]:
args.extend(["--profile=profiling_results.prof", "--savestats",])
# now we're committed
os.chdir(basedir)
from twisted.scripts import twistd
sys.argv = args
twistd.run()
def find_shares(options):
"""Given a storage index and a list of node directories, emit a list of
all matching shares to stdout, one per line. For example:
find-shares.py 44kai1tui348689nrw8fjegc8c ~/testnet/node-*
gives:
/home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/5
/home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9
/home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
"""
from allmydata.storage.server import si_a2b, storage_index_to_dir
from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path
out = options.stdout
sharedir = storage_index_to_dir(si_a2b(options.si_s.encode("utf-8")))
for d in options.nodedirs:
d = os.path.join(d, "storage", "shares", sharedir)
if os.path.exists(d):
for shnum in listdir_unicode(d):
print(quote_local_unicode_path(os.path.join(d, shnum),
quotemarks=False),
file=out)
return 0
def test_listdir_unicode(self):
if "dirlist" not in dir(self):
return
try:
u"test".encode(self.filesystem_encoding)
except (LookupError, AttributeError):
raise unittest.SkipTest(
"This platform does not support the '%s' filesystem encoding "
"that we are testing for the benefit of a different platform." % (self.filesystem_encoding,)
)
def call_os_listdir(path):
return self.dirlist
self.patch(os, "listdir", call_os_listdir)
def call_sys_getfilesystemencoding():
return self.filesystem_encoding
self.patch(sys, "getfilesystemencoding", call_sys_getfilesystemencoding)
_reload()
filenames = listdir_unicode(u"/dummy")
self.failUnlessEqual(set([normalize(fname) for fname in filenames]), set(TEST_FILENAMES))
def start(opts, out=sys.stdout, err=sys.stderr):
basedir = opts['basedir']
print >>out, "STARTING", quote_output(basedir)
if not os.path.isdir(basedir):
print >>err, "%s does not look like a directory at all" % quote_output(basedir)
return 1
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = str(fn)
break
else:
print >>err, "%s does not look like a node directory (no .tac file)" % quote_output(basedir)
return 1
if "client" in tac:
nodetype = "client"
elif "introducer" in tac:
nodetype = "introducer"
else:
nodetype = "unknown (%s)" % tac
args = ["twistd", "-y", tac]
if opts["syslog"]:
args.append("--syslog")
elif nodetype in ("client", "introducer"):
fileutil.make_dirs(os.path.join(basedir, "logs"))
args.extend(["--logfile", os.path.join("logs", "twistd.log")])
if opts["profile"]:
args.extend(["--profile=profiling_results.prof", "--savestats",])
# now we're committed
os.chdir(basedir)
from twisted.scripts import twistd
sys.argv = args
twistd.run()
def create_node(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_output(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
f = open(os.path.join(basedir, "tahoe-client.tac"), "w")
f.write(client_tac)
f.close()
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.write("[client]\n")
c.write("introducer.furl = %s\n" % config.get("introducer", ""))
c.write("helper.furl =\n")
c.write("#key_generator.furl =\n")
c.write("#stats_gatherer.furl =\n")
c.write("#shares.needed = 3\n")
c.write("#shares.happy = 7\n")
c.write("#shares.total = 10\n")
c.write("\n")
boolstr = {True:"true", False:"false"}
c.write("[storage]\n")
storage_enabled = not config.get("no-storage", None)
c.write("enabled = %s\n" % boolstr[storage_enabled])
c.write("#readonly =\n")
c.write("#reserved_space =\n")
c.write("#expire.enabled =\n")
c.write("#expire.mode =\n")
c.write("\n")
c.write("[helper]\n")
c.write("enabled = false\n")
c.write("\n")
c.close()
from allmydata.util import fileutil
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
print >>out, "Node created in %s" % quote_output(basedir)
if not config.get("introducer", ""):
print >>out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >>out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >>out, " Please set [node]nickname= in tahoe.cfg"
return 0
def test_no_unicode_normalization(self, mock):
# Pretend to run on a Unicode platform.
# We normalized to NFC in 1.7beta, but we now don't.
orig_platform = sys.platform
try:
sys.platform = 'darwin'
mock.return_value = [Artonwall_nfd]
_reload()
self.failUnlessReallyEqual(listdir_unicode(u'/dummy'), [Artonwall_nfd])
finally:
sys.platform = orig_platform
def test_no_unicode_normalization(self):
# Pretend to run on a Unicode platform.
# listdir_unicode normalized to NFC in 1.7beta, but now doesn't.
def call_os_listdir(path):
return [Artonwall_nfd]
self.patch(os, 'listdir', call_os_listdir)
self.patch(sys, 'platform', 'darwin')
_reload()
self.failUnlessReallyEqual(listdir_unicode(u'/dummy'), [Artonwall_nfd])
def test_no_unicode_normalization(self):
# Pretend to run on a Unicode platform.
# listdir_unicode normalized to NFC in 1.7beta, but now doesn't.
def call_os_listdir(path):
return [Artonwall_nfd]
self.patch(os, 'listdir', call_os_listdir)
self.patch(sys, 'platform', 'darwin')
_reload()
self.failUnlessReallyEqual(listdir_unicode(u'/dummy'), [Artonwall_nfd])
def test_no_unicode_normalization(self, mock):
# Pretend to run on a Unicode platform.
# We normalized to NFC in 1.7beta, but we now don't.
orig_platform = sys.platform
try:
sys.platform = 'darwin'
mock.return_value = [Artonwall_nfd]
_reload()
self.failUnlessReallyEqual(listdir_unicode(u'/dummy'), [Artonwall_nfd])
finally:
sys.platform = orig_platform
def identify_node_type(basedir):
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = str(fn)
break
else:
return None
for t in ("client", "introducer", "key-generator", "stats-gatherer"):
if t in tac:
return t
return None
def identify_node_type(basedir):
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = str(fn)
break
else:
return None
for t in ("client", "introducer", "key-generator", "stats-gatherer"):
if t in tac:
return t
return None
def populate(self, recurse):
if self.children is not None:
return
self.children = {}
children = listdir_unicode(self.pathname)
for i, n in enumerate(children):
self.progressfunc("examining %d of %d" % (i + 1, len(children)))
pn = os.path.join(self.pathname, n)
if os.path.isdir(pn):
child = LocalDirectoryTarget(self.progressfunc, pn)
self.children[n] = child
if recurse:
child.populate(recurse=True)
else:
assert os.path.isfile(pn)
self.children[n] = LocalFileTarget(pn)
def create_key_generator(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_output(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "key-generator")
return 0
def populate(self, recurse):
if self.children is not None:
return
self.children = {}
children = listdir_unicode(self.pathname)
for i,n in enumerate(children):
self.progressfunc("examining %d of %d" % (i+1, len(children)))
pn = os.path.join(self.pathname, n)
if os.path.isdir(pn):
child = LocalDirectoryTarget(self.progressfunc, pn)
self.children[n] = child
if recurse:
child.populate(recurse=True)
else:
assert os.path.isfile(pn)
self.children[n] = LocalFileTarget(pn)
def create_stats_gatherer(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_output(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "stats-gatherer")
return 0
def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err):
from allmydata.util.encodingutil import listdir_unicode, quote_output
try:
for shnum_s in sorted(listdir_unicode(si_dir), key=_as_number):
abs_sharefile = os.path.join(si_dir, shnum_s)
assert os.path.isfile(abs_sharefile)
try:
describe_share(abs_sharefile, si_s, shnum_s, now,
out)
except:
print >>err, "Error processing %s" % quote_output(abs_sharefile)
failure.Failure().printTraceback(err)
except:
print >>err, "Error processing %s" % quote_output(si_dir)
failure.Failure().printTraceback(err)
def catalog_shares_one_abbrevdir(si_s, si_dir, now, out, err):
from allmydata.util.encodingutil import listdir_unicode, quote_output
try:
for shnum_s in sorted(listdir_unicode(si_dir), key=_as_number):
abs_sharefile = os.path.join(si_dir, shnum_s)
assert os.path.isfile(abs_sharefile)
try:
describe_share(abs_sharefile, si_s, shnum_s, now, out)
except:
print("Error processing %s" % quote_output(abs_sharefile),
file=err)
failure.Failure().printTraceback(err)
except:
print("Error processing %s" % quote_output(si_dir), file=err)
failure.Failure().printTraceback(err)
def create_stats_gatherer(basedir, config, out=sys.stdout, err=sys.stderr):
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_output(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
f = open(os.path.join(basedir, "tahoe-stats-gatherer.tac"), "wb")
f.write(stats_gatherer_tac)
f.close()
return 0
def identify_node_type(basedir):
"""
:return unicode: None or one of: 'client' or 'introducer'.
"""
tac = u''
try:
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = fn
break
except OSError:
return None
for t in (u"client", u"introducer"):
if t in tac:
return t
return None
def identify_node_type(basedir):
"""
:return unicode: None or one of: 'client', 'introducer',
'key-generator' or 'stats-gatherer'
"""
tac = u''
try:
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = fn
break
except OSError:
return None
for t in (u"client", u"introducer", u"key-generator", u"stats-gatherer"):
if t in tac:
return t
return None
def identify_node_type(basedir):
"""
:return unicode: None or one of: 'client', 'introducer',
'key-generator' or 'stats-gatherer'
"""
tac = u''
try:
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = fn
break
except OSError:
return None
for t in (u"client", u"introducer", u"key-generator", u"stats-gatherer"):
if t in tac:
return t
return None
def test_listdir_unicode(self, mock_listdir, mock_getfilesystemencoding):
if 'dirlist' not in dir(self):
return
try:
u"test".encode(self.filesystem_encoding)
except (LookupError, AttributeError):
raise unittest.SkipTest("This platform does not support the '%s' filesystem encoding "
"that we are testing for the benefit of a different platform."
% (self.filesystem_encoding,))
mock_listdir.return_value = self.dirlist
mock_getfilesystemencoding.return_value = self.filesystem_encoding
_reload()
filenames = listdir_unicode(u'/dummy')
self.failUnlessEqual(set([normalize(fname) for fname in filenames]),
set(TEST_FILENAMES))
def test_listdir_unicode(self, mock_listdir, mock_getfilesystemencoding):
if 'dirlist' not in dir(self):
return
try:
u"test".encode(self.filesystem_encoding)
except (LookupError, AttributeError):
raise unittest.SkipTest("This platform does not support the '%s' filesystem encoding "
"that we are testing for the benefit of a different platform."
% (self.filesystem_encoding,))
mock_listdir.return_value = self.dirlist
mock_getfilesystemencoding.return_value = self.filesystem_encoding
_reload()
filenames = listdir_unicode(u'/dummy')
self.failUnlessEqual(set([normalize(fname) for fname in filenames]),
set(TEST_FILENAMES))
def populate(self, recurse):
if self.children is not None:
return
self.children = {}
children = listdir_unicode(self.pathname)
for i,n in enumerate(children):
self.progressfunc("examining %d of %d" % (i+1, len(children)))
pn = os.path.join(self.pathname, n)
if os.path.isdir(pn):
child = LocalDirectorySource(self.progressfunc, pn, n)
self.children[n] = child
if recurse:
child.populate(recurse=True)
elif os.path.isfile(pn):
self.children[n] = LocalFileSource(pn, n)
else:
# Could be dangling symlink; probably not copy-able.
# TODO: output a warning
pass
def populate(self, recurse):
if self.children is not None:
return
self.children = {}
children = listdir_unicode(self.pathname)
for i, n in enumerate(children):
self.progressfunc("examining %d of %d" % (i + 1, len(children)))
pn = os.path.join(self.pathname, n)
if os.path.isdir(pn):
child = LocalDirectorySource(self.progressfunc, pn)
self.children[n] = child
if recurse:
child.populate(True)
elif os.path.isfile(pn):
self.children[n] = LocalFileSource(pn)
else:
# Could be dangling symlink; probably not copy-able.
# TODO: output a warning
pass
def run(config, stdout, stderr):
from twisted.internet import reactor
from twisted.python import log, logfile
from allmydata import client
basedir = config['basedir']
precondition(isinstance(basedir, unicode), basedir)
if not os.path.isdir(basedir):
print >> stderr, "%s does not look like a directory at all" % quote_output(
basedir)
return 1
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = str(fn)
break
else:
print >> stderr, "%s does not look like a node directory (no .tac file)" % quote_output(
basedir)
return 1
if "client" not in tac:
print >> stderr, ("%s looks like it contains a non-client node (%s).\n"
"Use 'tahoe start' instead of 'tahoe run'." %
(quote_output(basedir), tac))
return 1
os.chdir(basedir)
# set up twisted logging. this will become part of the node rsn.
logdir = os.path.join(basedir, 'logs')
if not os.path.exists(logdir):
os.makedirs(logdir)
lf = logfile.LogFile('tahoesvc.log', logdir)
log.startLogging(lf)
# run the node itself
c = client.Client(basedir)
reactor.callLater(0, c.startService) # after reactor startup
reactor.run()
return 0
def run(config, stdout, stderr):
from twisted.internet import reactor
from twisted.python import log, logfile
from allmydata import client
basedir = config["basedir"]
precondition(isinstance(basedir, unicode), basedir)
if not os.path.isdir(basedir):
print >> stderr, "%s does not look like a directory at all" % quote_output(basedir)
return 1
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = str(fn)
break
else:
print >> stderr, "%s does not look like a node directory (no .tac file)" % quote_output(basedir)
return 1
if "client" not in tac:
print >> stderr, (
"%s looks like it contains a non-client node (%s).\n"
"Use 'tahoe start' instead of 'tahoe run'." % (quote_output(basedir), tac)
)
return 1
os.chdir(basedir)
# set up twisted logging. this will become part of the node rsn.
logdir = os.path.join(basedir, "logs")
if not os.path.exists(logdir):
os.makedirs(logdir)
lf = logfile.LogFile("tahoesvc.log", logdir)
log.startLogging(lf)
# run the node itself
c = client.Client(basedir)
reactor.callLater(0, c.startService) # after reactor startup
reactor.run()
return 0
def test_mkdir_open_exists_abspath_listdir_expanduser(self):
skip_if_cannot_represent_filename(lumiere_nfc)
try:
os.mkdir(lumiere_nfc)
except EnvironmentError as e:
raise unittest.SkipTest("%r\nIt is possible that the filesystem on which this test is being run "
"does not support Unicode, even though the platform does." % (e,))
fn = lumiere_nfc + u'/' + lumiere_nfc + u'.txt'
open(fn, 'wb').close()
self.failUnless(os.path.exists(fn))
self.failUnless(os.path.exists(os.path.join(os.getcwdu(), fn)))
filenames = listdir_unicode(lumiere_nfc)
# We only require that the listing includes a filename that is canonically equivalent
# to lumiere_nfc (on Mac OS X, it will be the NFD equivalent).
self.failUnlessIn(lumiere_nfc + ".txt", set([normalize(fname) for fname in filenames]))
expanded = fileutil.expanduser(u"~/" + lumiere_nfc)
self.failIfIn(u"~", expanded)
self.failUnless(expanded.endswith(lumiere_nfc), expanded)
def create_introducer(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "introducer")
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.close()
print >>out, "Introducer created in %s" % quote_local_unicode_path(basedir)
return 0
def create_introducer(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print("The base directory %s is not empty." % quote_local_unicode_path(basedir), file=err)
print("To avoid clobbering anything, I am going to quit now.", file=err)
print("Please use a different directory, or empty this one.", file=err)
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "introducer")
fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
print("Introducer created in %s" % quote_local_unicode_path(basedir), file=out)
defer.returnValue(0)
def find_shares(options):
"""Given a storage index and a list of node directories, emit a list of
all matching shares to stdout, one per line. For example:
find-shares.py 44kai1tui348689nrw8fjegc8c ~/testnet/node-*
gives:
/home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/5
/home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9
/home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
"""
from allmydata.storage.server import si_a2b, storage_index_to_dir
from allmydata.util.encodingutil import listdir_unicode
out = options.stdout
sharedir = storage_index_to_dir(si_a2b(options.si_s))
for d in options.nodedirs:
d = os.path.join(d, "storage/shares", sharedir)
if os.path.exists(d):
for shnum in listdir_unicode(d):
print >>out, os.path.join(d, shnum)
return 0
def create_node(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >> err, "The base directory %s is not empty." % quote_local_unicode_path(
basedir)
print >> err, "To avoid clobbering anything, I am going to quit now."
print >> err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.write("[client]\n")
c.write("# Which services should this client connect to?\n")
c.write("introducer.furl = %s\n" % config.get("introducer", ""))
c.write("helper.furl =\n")
c.write("#stats_gatherer.furl =\n")
c.write("\n")
c.write(
"# Encoding parameters this client will use for newly-uploaded files\n"
)
c.write("# This can be changed at any time: the encoding is saved in\n")
c.write(
"# each filecap, and we can download old files with any encoding\n")
c.write("# settings\n")
c.write("#shares.needed = 3\n")
c.write("#shares.happy = 7\n")
c.write("#shares.total = 10\n")
c.write("\n")
boolstr = {True: "true", False: "false"}
c.write("[storage]\n")
c.write("# Shall this node provide storage service?\n")
storage_enabled = not config.get("no-storage", None)
c.write("enabled = %s\n" % boolstr[storage_enabled])
c.write("#readonly =\n")
c.write("reserved_space = 1G\n")
c.write("#expire.enabled =\n")
c.write("#expire.mode =\n")
c.write("\n")
c.write("[helper]\n")
c.write("# Shall this node run a helper service that clients can use?\n")
c.write("enabled = false\n")
c.write("\n")
c.close()
from allmydata.util import fileutil
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
print >> out, "Node created in %s" % quote_local_unicode_path(basedir)
if not config.get("introducer", ""):
print >> out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >> out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >> out, " Please set [node]nickname= in tahoe.cfg"
return 0
def create_node(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
# if we're doing magic-wormhole stuff, do it now
if config['join'] is not None:
try:
remote_config = yield _get_config_via_wormhole(config)
except RuntimeError as e:
print >>err, str(e)
defer.returnValue(1)
# configuration we'll allow the inviter to set
whitelist = [
'shares-happy', 'shares-needed', 'shares-total',
'introducer', 'nickname',
]
sensitive_keys = ['introducer']
print >>out, "Encoding: {shares-needed} of {shares-total} shares, on at least {shares-happy} servers".format(**remote_config)
print >>out, "Overriding the following config:"
for k in whitelist:
v = remote_config.get(k, None)
if v is not None:
# we're faking usually argv-supplied options :/
if isinstance(v, unicode):
v = v.encode(get_io_encoding())
config[k] = v
if k not in sensitive_keys:
if k not in ['shares-happy', 'shares-total', 'shares-needed']:
print >>out, " {}: {}".format(k, v)
else:
print >>out, " {}: [sensitive data; see tahoe.cfg]".format(k)
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
write_client_config(c, config)
print >>out, "Node created in %s" % quote_local_unicode_path(basedir)
tahoe_cfg = quote_local_unicode_path(os.path.join(basedir, "tahoe.cfg"))
if not config.get("introducer", ""):
print >>out, " Please set [client]introducer.furl= in %s!" % tahoe_cfg
print >>out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >>out, " Please set [node]nickname= in %s" % tahoe_cfg
defer.returnValue(0)
def create_node(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >> err, "The base directory %s is not empty." % quote_local_unicode_path(
basedir)
print >> err, "To avoid clobbering anything, I am going to quit now."
print >> err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
# if we're doing magic-wormhole stuff, do it now
if config['join'] is not None:
try:
remote_config = yield _get_config_via_wormhole(config)
except RuntimeError as e:
print >> err, str(e)
defer.returnValue(1)
# configuration we'll allow the inviter to set
whitelist = [
'shares-happy',
'shares-needed',
'shares-total',
'introducer',
'nickname',
]
sensitive_keys = ['introducer']
print >> out, "Encoding: {shares-needed} of {shares-total} shares, on at least {shares-happy} servers".format(
**remote_config)
print >> out, "Overriding the following config:"
for k in whitelist:
v = remote_config.get(k, None)
if v is not None:
# we're faking usually argv-supplied options :/
if isinstance(v, unicode):
v = v.encode(get_io_encoding())
config[k] = v
if k not in sensitive_keys:
if k not in [
'shares-happy', 'shares-total', 'shares-needed'
]:
print >> out, " {}: {}".format(k, v)
else:
print >> out, " {}: [sensitive data; see tahoe.cfg]".format(
k)
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
write_client_config(c, config)
print >> out, "Node created in %s" % quote_local_unicode_path(basedir)
tahoe_cfg = quote_local_unicode_path(os.path.join(basedir, "tahoe.cfg"))
if not config.get("introducer", ""):
print >> out, " Please set [client]introducer.furl= in %s!" % tahoe_cfg
print >> out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >> out, " Please set [node]nickname= in %s" % tahoe_cfg
defer.returnValue(0)
def create_node(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.write("[client]\n")
c.write("# Which services should this client connect to?\n")
c.write("introducer.furl = %s\n" % config.get("introducer", ""))
c.write("helper.furl =\n")
c.write("#key_generator.furl =\n")
c.write("#stats_gatherer.furl =\n")
c.write("\n")
c.write("# What encoding parameters should this client use for uploads?\n")
c.write("#shares.needed = 3\n")
c.write("#shares.happy = 7\n")
c.write("#shares.total = 10\n")
c.write("\n")
boolstr = {True:"true", False:"false"}
c.write("[storage]\n")
c.write("# Shall this node provide storage service?\n")
storage_enabled = not config.get("no-storage", None)
c.write("enabled = %s\n" % boolstr[storage_enabled])
c.write("#readonly =\n")
c.write("reserved_space = 1G\n")
c.write("#expire.enabled =\n")
c.write("#expire.mode =\n")
c.write("\n")
c.write("[helper]\n")
c.write("# Shall this node run a helper service that clients can use?\n")
c.write("enabled = false\n")
c.write("\n")
c.write("[drop_upload]\n")
c.write("# Shall this node automatically upload files created or modified in a local directory?\n")
c.write("enabled = false\n")
c.write("# To specify the target of uploads, a mutable directory writecap URI must be placed\n"
"# in 'private/drop_upload_dircap'.\n")
c.write("local.directory = ~/drop_upload\n")
c.write("\n")
c.close()
from allmydata.util import fileutil
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
print >>out, "Node created in %s" % quote_local_unicode_path(basedir)
if not config.get("introducer", ""):
print >>out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >>out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >>out, " Please set [node]nickname= in tahoe.cfg"
return 0
def create_node(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >> err, "The base directory %s is not empty." % quote_output(
basedir)
print >> err, "To avoid clobbering anything, I am going to quit now."
print >> err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
f = open(os.path.join(basedir, "tahoe-client.tac"), "w")
f.write(client_tac)
f.close()
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.write("[client]\n")
c.write("# Which services should this client connect to?\n")
c.write("introducer.furl = %s\n" % config.get("introducer", ""))
c.write("helper.furl =\n")
c.write("#key_generator.furl =\n")
c.write("#stats_gatherer.furl =\n")
c.write("\n")
c.write("# What encoding parameters should this client use for uploads?\n")
c.write("#shares.needed = 3\n")
c.write("#shares.happy = 7\n")
c.write("#shares.total = 10\n")
c.write("\n")
boolstr = {True: "true", False: "false"}
c.write("[storage]\n")
c.write("# Shall this node provide storage service?\n")
storage_enabled = not config.get("no-storage", None)
c.write("enabled = %s\n" % boolstr[storage_enabled])
c.write("#readonly =\n")
c.write("reserved_space = 1G\n")
c.write("#expire.enabled =\n")
c.write("#expire.mode =\n")
c.write("\n")
c.write("[helper]\n")
c.write("# Shall this node run a helper service that clients can use?\n")
c.write("enabled = false\n")
c.write("\n")
c.write("[drop_upload]\n")
c.write(
"# Shall this node automatically upload files created or modified in a local directory?\n"
)
c.write("enabled = false\n")
c.write(
"# To specify the target of uploads, a mutable directory writecap URI must be placed\n"
"# in 'private/drop_upload_dircap'.\n")
c.write("local.directory = ~/drop_upload\n")
c.write("\n")
c.close()
from allmydata.util import fileutil
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
print >> out, "Node created in %s" % quote_output(basedir)
if not config.get("introducer", ""):
print >> out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >> out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >> out, " Please set [node]nickname= in tahoe.cfg"
return 0
def create_node(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.write("[client]\n")
c.write("# Which services should this client connect to?\n")
c.write("introducer.furl = %s\n" % config.get("introducer", ""))
c.write("helper.furl =\n")
c.write("#stats_gatherer.furl =\n")
c.write("\n")
c.write("# Encoding parameters this client will use for newly-uploaded files\n")
c.write("# This can be changed at any time: the encoding is saved in\n")
c.write("# each filecap, and we can download old files with any encoding\n")
c.write("# settings\n")
c.write("#shares.needed = 3\n")
c.write("#shares.happy = 7\n")
c.write("#shares.total = 10\n")
c.write("\n")
boolstr = {True:"true", False:"false"}
c.write("[storage]\n")
c.write("# Shall this node provide storage service?\n")
storage_enabled = not config.get("no-storage", None)
c.write("enabled = %s\n" % boolstr[storage_enabled])
c.write("#readonly =\n")
c.write("reserved_space = 1G\n")
c.write("#expire.enabled =\n")
c.write("#expire.mode =\n")
c.write("\n")
c.write("[helper]\n")
c.write("# Shall this node run a helper service that clients can use?\n")
c.write("enabled = false\n")
c.write("\n")
c.close()
from allmydata.util import fileutil
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
print >>out, "Node created in %s" % quote_local_unicode_path(basedir)
if not config.get("introducer", ""):
print >>out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >>out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >>out, " Please set [node]nickname= in tahoe.cfg"
return 0
def do_start(basedir, opts, out=sys.stdout, err=sys.stderr):
print >>out, "STARTING", quote_output(basedir)
if not os.path.isdir(basedir):
print >>err, "%s does not look like a directory at all" % quote_output(basedir)
return 1
for fn in listdir_unicode(basedir):
if fn.endswith(u".tac"):
tac = str(fn)
break
else:
print >>err, "%s does not look like a node directory (no .tac file)" % quote_output(basedir)
return 1
if "client" in tac:
nodetype = "client"
elif "introducer" in tac:
nodetype = "introducer"
else:
nodetype = "unknown (%s)" % tac
cmd = find_exe.find_exe('twistd')
if not cmd:
# If 'twistd' wasn't on $PATH, maybe we're running from source and
# Twisted was built as one of our dependencies. If so, we're at
# BASEDIR/src/allmydata/scripts/startstop_node.py, and it's at
# BASEDIR/support/$BINDIR/twistd
up = os.path.dirname
TAHOEDIR = up(up(up(up(os.path.abspath(__file__)))))
if sys.platform == "win32":
bin_dir = "Scripts"
else:
bin_dir = "bin"
bindir = os.path.join(TAHOEDIR, "support", bin_dir)
maybe = os.path.join(bindir, "twistd")
if os.path.exists(maybe):
cmd = [maybe]
oldpath = os.environ.get("PATH", "").split(os.pathsep)
os.environ["PATH"] = os.pathsep.join(oldpath + [bindir])
# sys.path and $PYTHONPATH are taken care of by the extra code in
# 'setup.py trial'
else:
maybe = maybe+'.py'
if os.path.exists(maybe):
cmd = [sys.executable, maybe]
oldpath = os.environ.get("PATH", "").split(os.pathsep)
os.environ["PATH"] = os.pathsep.join(oldpath + [bindir])
# sys.path and $PYTHONPATH are taken care of by the extra code in
# 'setup.py trial'
if not cmd:
print "Can't find twistd (it comes with Twisted). Aborting."
sys.exit(1)
cmd.extend(["-y", tac])
if opts["syslog"]:
cmd.append("--syslog")
elif nodetype in ("client", "introducer"):
fileutil.make_dirs(os.path.join(basedir, "logs"))
cmd.extend(["--logfile", os.path.join("logs", "twistd.log")])
if opts["profile"]:
cmd.extend(["--profile=profiling_results.prof", "--savestats",])
curdir = os.getcwd()
try:
os.chdir(basedir)
rc = os.system(' '.join(cmd))
finally:
os.chdir(curdir)
if rc == 0:
print >>out, "%s node probably started" % nodetype
return 0
else:
print >>err, "%s node probably not started" % nodetype
return 1
