def create_node(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >> err, "The base directory %s is not empty." % quote_local_unicode_path(
basedir)
print >> err, "To avoid clobbering anything, I am going to quit now."
print >> err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
write_client_config(c, config)
print >> out, "Node created in %s" % quote_local_unicode_path(basedir)
tahoe_cfg = quote_local_unicode_path(os.path.join(basedir, "tahoe.cfg"))
if not config.get("introducer", ""):
print >> out, " Please set [client]introducer.furl= in %s!" % tahoe_cfg
print >> out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >> out, " Please set [node]nickname= in %s" % tahoe_cfg
defer.returnValue(0)
def create_node(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
write_client_config(c, config)
print >>out, "Node created in %s" % quote_local_unicode_path(basedir)
if not config.get("introducer", ""):
print >>out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >>out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >>out, " Please set [node]nickname= in tahoe.cfg"
defer.returnValue(0)
def __init__(self, client, local_path_u, db, name, clock, delay=0):
self._client = client
self._local_path_u = local_path_u
self._local_filepath = to_filepath(local_path_u)
self._db = db
self._name = name
self._clock = clock
self._hooks = {
'processed': None,
'started': None,
'iteration': None,
}
self.started_d = self.set_hook('started')
if not self._local_filepath.exists():
raise AssertionError("The '[magic_folder] local.directory' parameter was %s "
"but there is no directory at that location."
% quote_local_unicode_path(self._local_path_u))
if not self._local_filepath.isdir():
raise AssertionError("The '[magic_folder] local.directory' parameter was %s "
"but the thing at that location is not a directory."
% quote_local_unicode_path(self._local_path_u))
self._deque = deque()
# do we also want to bound on "maximum age"?
self._process_history = deque(maxlen=20)
self._stopped = False
# XXX pass in an initial value for this; it seems like .10 broke this and it's always 0
self._turn_delay = delay
self._log('delay is %f' % self._turn_delay)
# a Deferred to wait for the _do_processing() loop to exit
# (gets set to the return from _do_processing() if we get that
# far)
self._processing = defer.succeed(None)
def upload(self, childpath):
precondition_abspath(childpath)
#self.verboseprint("uploading %s.." % quote_local_unicode_path(childpath))
metadata = get_local_metadata(childpath)
# we can use the backupdb here
must_upload, bdb_results = self.check_backupdb_file(childpath)
if must_upload:
self.verboseprint("uploading %s.." %
quote_local_unicode_path(childpath))
infileobj = open(childpath, "rb")
url = self.options['node-url'] + "uri"
resp = do_http("PUT", url, infileobj)
if resp.status not in (200, 201):
raise HTTPError("Error during file PUT", resp)
filecap = resp.read().strip()
self.verboseprint(
" %s -> %s" %
(quote_local_unicode_path(childpath, quotemarks=False),
quote_output(filecap, quotemarks=False)))
#self.verboseprint(" metadata: %s" % (quote_output(metadata, quotemarks=False),))
if bdb_results:
bdb_results.did_upload(filecap)
return True, filecap, metadata
else:
self.verboseprint("skipping %s.." %
quote_local_unicode_path(childpath))
return False, bdb_results.was_uploaded(), metadata
def create_introducer(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >> err, "The base directory %s is not empty." % quote_local_unicode_path(
basedir)
print >> err, "To avoid clobbering anything, I am going to quit now."
print >> err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "introducer")
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
print >> out, "Introducer created in %s" % quote_local_unicode_path(
basedir)
defer.returnValue(0)
def upload(self, childpath):
precondition_abspath(childpath)
#self.verboseprint("uploading %s.." % quote_local_unicode_path(childpath))
metadata = get_local_metadata(childpath)
# we can use the backupdb here
must_upload, bdb_results = self.check_backupdb_file(childpath)
if must_upload:
self.verboseprint("uploading %s.." % quote_local_unicode_path(childpath))
infileobj = open(childpath, "rb")
url = self.options['node-url'] + "uri"
resp = do_http("PUT", url, infileobj)
if resp.status not in (200, 201):
raise HTTPError("Error during file PUT", resp)
filecap = resp.read().strip()
self.verboseprint(" %s -> %s" % (quote_local_unicode_path(childpath, quotemarks=False),
quote_output(filecap, quotemarks=False)))
#self.verboseprint(" metadata: %s" % (quote_output(metadata, quotemarks=False),))
if bdb_results:
bdb_results.did_upload(filecap)
return True, filecap, metadata
else:
self.verboseprint("skipping %s.." % quote_local_unicode_path(childpath))
return False, bdb_results.was_uploaded(), metadata
def __init__(self, client, local_path_u, db, name, clock):
self._client = client
self._local_path_u = local_path_u
self._local_filepath = to_filepath(local_path_u)
self._db = db
self._name = name
self._clock = clock
self._debug_log = False
self._logger = None
self._hooks = {
'processed': None,
'started': None,
'iteration': None,
'inotify': None,
}
self.started_d = self.set_hook('started')
if not self._local_filepath.exists():
raise AssertionError("The '[magic_folder] local.directory' parameter was %s "
"but there is no directory at that location."
% quote_local_unicode_path(self._local_path_u))
if not self._local_filepath.isdir():
raise AssertionError("The '[magic_folder] local.directory' parameter was %s "
"but the thing at that location is not a directory."
% quote_local_unicode_path(self._local_path_u))
self._deque = deque()
# do we also want to bound on "maximum age"?
self._process_history = deque(maxlen=20)
self._stopped = False
# a Deferred to wait for the _do_processing() loop to exit
# (gets set to the return from _do_processing() if we get that
# far)
self._processing = defer.succeed(None)
def upload_directory(self, path, compare_contents, create_contents):
must_create, r = self.check_backupdb_directory(compare_contents)
if must_create:
self.verboseprint(" creating directory for %s" % quote_local_unicode_path(path))
newdircap = mkdir(create_contents, self.options)
assert isinstance(newdircap, str)
if r:
r.did_create(newdircap)
return True, newdircap
else:
self.verboseprint(" re-using old directory for %s" % quote_local_unicode_path(path))
return False, r.was_created()
def upload_directory(self, path, compare_contents, create_contents):
must_create, r = self.check_backupdb_directory(compare_contents)
if must_create:
self.verboseprint(" creating directory for %s" % quote_local_unicode_path(path))
newdircap = mkdir(create_contents, self.options)
assert isinstance(newdircap, str)
if r:
r.did_create(newdircap)
return True, newdircap
else:
self.verboseprint(" re-using old directory for %s" % quote_local_unicode_path(path))
return False, r.was_created()
def _leave(node_directory, name, existing_folders):
privdir = os.path.join(node_directory, u"private")
db_fname = os.path.join(privdir, u"magicfolder_{}.sqlite".format(name))
# delete from YAML file and re-write it
del existing_folders[name]
save_magic_folders(node_directory, existing_folders)
# delete the database file
try:
fileutil.remove(db_fname)
except Exception as e:
raise Exception(
"unable to remove %s due to %s: %s" %
(quote_local_unicode_path(db_fname), e.__class__.__name__, str(e)))
# if this was the last magic-folder, disable them entirely
if not existing_folders:
parser = SafeConfigParser()
parser.read(os.path.join(node_directory, u"tahoe.cfg"))
parser.remove_section("magic_folder")
with open(os.path.join(node_directory, u"tahoe.cfg"), "w") as f:
parser.write(f)
return 0
def leave(options):
from ConfigParser import SafeConfigParser
dmd_cap_file = os.path.join(options["node-directory"], u"private",
u"magic_folder_dircap")
collective_readcap_file = os.path.join(options["node-directory"],
u"private", u"collective_dircap")
magic_folder_db_file = os.path.join(options["node-directory"], u"private",
u"magicfolderdb.sqlite")
parser = SafeConfigParser()
parser.read(os.path.join(options["node-directory"], u"tahoe.cfg"))
parser.remove_section("magic_folder")
f = open(os.path.join(options["node-directory"], u"tahoe.cfg"), "w")
parser.write(f)
f.close()
for f in [dmd_cap_file, collective_readcap_file, magic_folder_db_file]:
try:
fileutil.remove(f)
except Exception as e:
print >> options.stderr, (
"Warning: unable to remove %s due to %s: %s" %
(quote_local_unicode_path(f), e.__class__.__name__, str(e)))
# if this doesn't return 0, then the CLI stuff fails
return 0
def stop(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >>out, "STOPPING", quoted_basedir
pidfile = os.path.join(basedir, u"twistd.pid")
if not os.path.exists(pidfile):
print >>err, "%s does not look like a running node directory (no twistd.pid)" % quoted_basedir
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return 2
pid = open(pidfile, "r").read()
pid = int(pid)
# kill it hard (SIGKILL), delete the twistd.pid file, then wait for the
# process itself to go away. If it hasn't gone away after 20 seconds, warn
# the user but keep waiting until they give up.
try:
os.kill(pid, signal.SIGKILL)
except OSError, oserr:
if oserr.errno == 3:
print oserr.strerror
# the process didn't exist, so wipe the pid file
os.remove(pidfile)
return 2
else:
raise
def find_shares(options):
"""Given a storage index and a list of node directories, emit a list of
all matching shares to stdout, one per line. For example:
find-shares.py 44kai1tui348689nrw8fjegc8c ~/testnet/node-*
gives:
/home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/5
/home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9
/home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
"""
from allmydata.storage.server import si_a2b, storage_index_to_dir
from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path
out = options.stdout
sharedir = storage_index_to_dir(si_a2b(options.si_s.encode("utf-8")))
for d in options.nodedirs:
d = os.path.join(d, "storage", "shares", sharedir)
if os.path.exists(d):
for shnum in listdir_unicode(d):
print(quote_local_unicode_path(os.path.join(d, shnum),
quotemarks=False),
file=out)
return 0
def start(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >>out, "STARTING", quoted_basedir
if not os.path.isdir(basedir):
print >>err, "%s does not look like a directory at all" % quoted_basedir
return 1
nodetype = identify_node_type(basedir)
if not nodetype:
print >>err, "%s is not a recognizable node directory" % quoted_basedir
return 1
# Now prepare to turn into a twistd process. This os.chdir is the point
# of no return.
os.chdir(basedir)
twistd_args = []
if (nodetype in ("client", "introducer")
and "--nodaemon" not in config.twistd_args
and "--syslog" not in config.twistd_args
and "--logfile" not in config.twistd_args):
fileutil.make_dirs(os.path.join(basedir, u"logs"))
twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")])
twistd_args.extend(config.twistd_args)
twistd_args.append("StartTahoeNode") # point at our StartTahoeNodePlugin
twistd_config = MyTwistdConfig()
try:
twistd_config.parseOptions(twistd_args)
except usage.error, ue:
# these arguments were unsuitable for 'twistd'
print >>err, config
print >>err, "tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue)
return 1
class StartOptions(BasedirOptions):
subcommand_name = "start"
optParameters = [
("basedir", "C", None,
"Specify which Tahoe base directory should be used."
" This has the same effect as the global --node-directory option."
" [default: %s]" % quote_local_unicode_path(_default_nodedir)),
]
def parseArgs(self, basedir=None, *twistd_args):
# This can't handle e.g. 'tahoe start --nodaemon', since '--nodaemon'
# looks like an option to the tahoe subcommand, not to twistd. So you
# can either use 'tahoe start' or 'tahoe start NODEDIR
# --TWISTD-OPTIONS'. Note that 'tahoe --node-directory=NODEDIR start
# --TWISTD-OPTIONS' also isn't allowed, unfortunately.
BasedirOptions.parseArgs(self, basedir)
self.twistd_args = twistd_args
def getSynopsis(self):
return ("Usage: %s [global-options] %s [options]"
" [NODEDIR [twistd-options]]"
% (self.command_name, self.subcommand_name))
def getUsage(self, width=None):
t = BasedirOptions.getUsage(self, width) + "\n"
twistd_options = str(MyTwistdConfig()).partition("\n")[2].partition("\n\n")[0]
t += twistd_options.replace("Options:", "twistd-options:", 1)
t += """
Note that if any twistd-options are used, NODEDIR must be specified explicitly
(not by default or using -C/--basedir or -d/--node-directory), and followed by
the twistd-options.
"""
return t
def start(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >> out, "STARTING", quoted_basedir
if not os.path.isdir(basedir):
print >> err, "%s does not look like a directory at all" % quoted_basedir
return 1
nodetype = identify_node_type(basedir)
if not nodetype:
print >> err, "%s is not a recognizable node directory" % quoted_basedir
return 1
# Now prepare to turn into a twistd process. This os.chdir is the point
# of no return.
os.chdir(basedir)
twistd_args = []
if (nodetype in ("client", "introducer")
and "--nodaemon" not in config.twistd_args
and "--syslog" not in config.twistd_args
and "--logfile" not in config.twistd_args):
fileutil.make_dirs(os.path.join(basedir, u"logs"))
twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")])
twistd_args.extend(config.twistd_args)
twistd_args.append("StartTahoeNode") # point at our StartTahoeNodePlugin
twistd_config = MyTwistdConfig()
try:
twistd_config.parseOptions(twistd_args)
except usage.error, ue:
# these arguments were unsuitable for 'twistd'
print >> err, config
print >> err, "tahoe %s: usage error from twistd: %s\n" % (
config.subcommand_name, ue)
return 1
class BasedirOptions(BaseOptions):
default_nodedir = _default_nodedir
optParameters = [
["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]"
% quote_local_unicode_path(_default_nodedir)],
]
def parseArgs(self, basedir=None):
if self.parent['node-directory'] and self['basedir']:
raise usage.UsageError("The --node-directory (or -d) and --basedir (or -C) options cannot both be used.")
if self.parent['node-directory'] and basedir:
raise usage.UsageError("The --node-directory (or -d) option and a basedir argument cannot both be used.")
if self['basedir'] and basedir:
raise usage.UsageError("The --basedir (or -C) option and a basedir argument cannot both be used.")
if basedir:
b = argv_to_abspath(basedir)
elif self['basedir']:
b = argv_to_abspath(self['basedir'])
elif self.parent['node-directory']:
b = argv_to_abspath(self.parent['node-directory'])
elif self.default_nodedir:
b = self.default_nodedir
else:
raise usage.UsageError("No default basedir available, you must provide one with --node-directory, --basedir, or a basedir argument")
self['basedir'] = b
def postOptions(self):
if not self['basedir']:
raise usage.UsageError("A base directory for the node must be provided.")
class CreateClientOptions(_CreateBaseOptions):
synopsis = "[options] [NODEDIR]"
description = "Create a client-only Tahoe-LAFS node (no storage server)."
optParameters = [
# we provide 'create-node'-time options for the most common
# configuration knobs. The rest can be controlled by editing
# tahoe.cfg before node startup.
("nickname", "n", None, "Specify the nickname for this node."),
("introducer", "i", None, "Specify the introducer FURL to use."),
("webport", "p", "tcp:3456:interface=127.0.0.1",
"Specify which TCP port to run the HTTP interface on. Use 'none' to disable."
),
("basedir", "C", None,
"Specify which Tahoe base directory should be used. This has the same effect as the global --node-directory option. [default: %s]"
% quote_local_unicode_path(_default_nodedir)),
("shares-needed", None, 3,
"Needed shares required for uploaded files."),
("shares-happy", None, 7,
"How many servers new files must be placed on."),
("shares-total", None, 10,
"Total shares required for uploaded files."),
]
# This is overridden in order to ensure we get a "Wrong number of
# arguments." error when more than one argument is given.
def parseArgs(self, basedir=None):
BasedirOptions.parseArgs(self, basedir)
for name in ["shares-needed", "shares-happy", "shares-total"]:
try:
int(self[name])
except ValueError:
raise UsageError("--{} must be an integer".format(name))
def stop(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >> out, "STOPPING", quoted_basedir
pidfile = get_pidfile(basedir)
pid = get_pid_from_pidfile(pidfile)
if pid is None:
print >> err, "%s does not look like a running node directory (no twistd.pid)" % quoted_basedir
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return 2
elif pid == -1:
print >> err, "%s contains an invalid PID file" % basedir
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return 2
# kill it hard (SIGKILL), delete the twistd.pid file, then wait for the
# process itself to go away. If it hasn't gone away after 20 seconds, warn
# the user but keep waiting until they give up.
try:
os.kill(pid, signal.SIGKILL)
except OSError, oserr:
if oserr.errno == 3:
print oserr.strerror
# the process didn't exist, so wipe the pid file
os.remove(pidfile)
return COULD_NOT_STOP
else:
raise
def stop(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >> out, "STOPPING", quoted_basedir
pidfile = os.path.join(basedir, u"twistd.pid")
if not os.path.exists(pidfile):
print >> err, "%s does not look like a running node directory (no twistd.pid)" % quoted_basedir
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return 2
with open(pidfile, "r") as f:
pid = f.read()
pid = int(pid)
# kill it hard (SIGKILL), delete the twistd.pid file, then wait for the
# process itself to go away. If it hasn't gone away after 20 seconds, warn
# the user but keep waiting until they give up.
try:
os.kill(pid, signal.SIGKILL)
except OSError, oserr:
if oserr.errno == 3:
print oserr.strerror
# the process didn't exist, so wipe the pid file
os.remove(pidfile)
return 2
else:
raise
def run(config):
"""
Runs a Tahoe-LAFS node in the foreground.
Sets up the IService instance corresponding to the type of node
that's starting and uses Twisted's twistd runner to disconnect our
process from the terminal.
"""
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print("'tahoe {}' in {}".format(config.subcommand_name, quoted_basedir),
file=out)
if not os.path.isdir(basedir):
print("%s does not look like a directory at all" % quoted_basedir,
file=err)
return 1
nodetype = identify_node_type(basedir)
if not nodetype:
print("%s is not a recognizable node directory" % quoted_basedir,
file=err)
return 1
# Now prepare to turn into a twistd process. This os.chdir is the point
# of no return.
os.chdir(basedir)
twistd_args = ["--nodaemon"]
twistd_args.extend(config.twistd_args)
twistd_args.append(
"DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin
twistd_config = MyTwistdConfig()
twistd_config.stdout = out
twistd_config.stderr = err
try:
twistd_config.parseOptions(twistd_args)
except usage.error as ue:
# these arguments were unsuitable for 'twistd'
print(config, file=err)
print("tahoe %s: usage error from twistd: %s\n" %
(config.subcommand_name, ue),
file=err)
return 1
twistd_config.loadedPlugins = {
"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)
}
# handle invalid PID file (twistd might not start otherwise)
pidfile = get_pidfile(basedir)
if get_pid_from_pidfile(pidfile) == -1:
print("found invalid PID file in %s - deleting it" % basedir, file=err)
os.remove(pidfile)
# We always pass --nodaemon so twistd.runApp does not daemonize.
print("running node in %s" % (quoted_basedir, ), file=out)
twistd.runApp(twistd_config)
return 0
def process(self, localpath):
precondition_abspath(localpath)
# returns newdircap
quoted_path = quote_local_unicode_path(localpath)
self.verboseprint("processing %s" % (quoted_path,))
create_contents = {} # childname -> (type, rocap, metadata)
compare_contents = {} # childname -> rocap
try:
children = listdir_unicode(localpath)
except EnvironmentError:
self.directories_skipped += 1
self.warn("WARNING: permission denied on directory %s" % (quoted_path,))
children = []
except FilenameEncodingError:
self.directories_skipped += 1
self.warn("WARNING: could not list directory %s due to a filename encoding error" % (quoted_path,))
children = []
for child in self.options.filter_listdir(children):
assert isinstance(child, unicode), child
childpath = os.path.join(localpath, child)
# note: symlinks to directories are both islink() and isdir()
if os.path.isdir(childpath) and not os.path.islink(childpath):
metadata = get_local_metadata(childpath)
# recurse on the child directory
childcap = self.process(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("dirnode", childcap, metadata)
compare_contents[child] = childcap
elif os.path.isfile(childpath) and not os.path.islink(childpath):
try:
childcap, metadata = self.upload(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("filenode", childcap, metadata)
compare_contents[child] = childcap
except EnvironmentError:
self.files_skipped += 1
self.warn("WARNING: permission denied on file %s" % quote_local_unicode_path(childpath))
else:
self.files_skipped += 1
if os.path.islink(childpath):
self.warn("WARNING: cannot backup symlink %s" % quote_local_unicode_path(childpath))
else:
self.warn("WARNING: cannot backup special file %s" % quote_local_unicode_path(childpath))
must_create, r = self.check_backupdb_directory(compare_contents)
if must_create:
self.verboseprint(" creating directory for %s" % quote_local_unicode_path(localpath))
newdircap = mkdir(create_contents, self.options)
assert isinstance(newdircap, str)
if r:
r.did_create(newdircap)
self.directories_created += 1
return newdircap
else:
self.verboseprint(" re-using old directory for %s" % quote_local_unicode_path(localpath))
self.directories_reused += 1
return r.was_created()
def process(self, localpath):
precondition_abspath(localpath)
# returns newdircap
quoted_path = quote_local_unicode_path(localpath)
self.verboseprint("processing %s" % (quoted_path,))
create_contents = {} # childname -> (type, rocap, metadata)
compare_contents = {} # childname -> rocap
try:
children = listdir_unicode(localpath)
except EnvironmentError:
self.directories_skipped += 1
self.warn("WARNING: permission denied on directory %s" % (quoted_path,))
children = []
except FilenameEncodingError:
self.directories_skipped += 1
self.warn("WARNING: could not list directory %s due to a filename encoding error" % (quoted_path,))
children = []
for child in self.options.filter_listdir(children):
assert isinstance(child, unicode), child
childpath = os.path.join(localpath, child)
# note: symlinks to directories are both islink() and isdir()
if os.path.isdir(childpath) and not os.path.islink(childpath):
metadata = get_local_metadata(childpath)
# recurse on the child directory
childcap = self.process(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("dirnode", childcap, metadata)
compare_contents[child] = childcap
elif os.path.isfile(childpath) and not os.path.islink(childpath):
try:
childcap, metadata = self.upload(childpath)
assert isinstance(childcap, str)
create_contents[child] = ("filenode", childcap, metadata)
compare_contents[child] = childcap
except EnvironmentError:
self.files_skipped += 1
self.warn("WARNING: permission denied on file %s" % quote_local_unicode_path(childpath))
else:
self.files_skipped += 1
if os.path.islink(childpath):
self.warn("WARNING: cannot backup symlink %s" % quote_local_unicode_path(childpath))
else:
self.warn("WARNING: cannot backup special file %s" % quote_local_unicode_path(childpath))
must_create, r = self.check_backupdb_directory(compare_contents)
if must_create:
self.verboseprint(" creating directory for %s" % quote_local_unicode_path(localpath))
newdircap = mkdir(create_contents, self.options)
assert isinstance(newdircap, str)
if r:
r.did_create(newdircap)
self.directories_created += 1
return newdircap
else:
self.verboseprint(" re-using old directory for %s" % quote_local_unicode_path(localpath))
self.directories_reused += 1
return r.was_created()
def test_quote_path(self):
self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), b"'foo/bar'")
self.failUnlessReallyEqual(
quote_path([u'foo', u'bar'], quotemarks=True), b"'foo/bar'")
self.failUnlessReallyEqual(
quote_path([u'foo', u'bar'], quotemarks=False), b"foo/bar")
self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']),
b'"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_path([u'foo', u'\nbar'], quotemarks=True), b'"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_path([u'foo', u'\nbar'], quotemarks=False),
b'"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo"),
win32_other(b"'C:\\foo'", b"'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True),
win32_other(b"'C:\\foo'", b"'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False),
win32_other(b"C:\\foo", b"\\\\?\\C:\\foo"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"),
win32_other(b"'\\\\foo\\bar'", b"'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True),
win32_other(b"'\\\\foo\\bar'", b"'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar",
quotemarks=False),
win32_other(b"\\\\foo\\bar", b"\\\\?\\UNC\\foo\\bar"))
def test_quote_path(self):
self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), "'foo/bar'")
self.failUnlessReallyEqual(
quote_path([u'foo', u'bar'], quotemarks=True), "'foo/bar'")
self.failUnlessReallyEqual(
quote_path([u'foo', u'bar'], quotemarks=False), "foo/bar")
self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']),
'"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_path([u'foo', u'\nbar'], quotemarks=True), '"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_path([u'foo', u'\nbar'], quotemarks=False), '"foo/\\x0abar"')
def win32_other(win32, other):
return win32 if sys.platform == "win32" else other
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo"),
win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True),
win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False),
win32_other("C:\\foo", "\\\\?\\C:\\foo"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"),
win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True),
win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar",
quotemarks=False),
win32_other("\\\\foo\\bar", "\\\\?\\UNC\\foo\\bar"))
def test_quote_path(self):
self.failUnlessReallyEqual(quote_path([u"foo", u"bar"]), "'foo/bar'")
self.failUnlessReallyEqual(quote_path([u"foo", u"bar"], quotemarks=True), "'foo/bar'")
self.failUnlessReallyEqual(quote_path([u"foo", u"bar"], quotemarks=False), "foo/bar")
self.failUnlessReallyEqual(quote_path([u"foo", u"\nbar"]), '"foo/\\x0abar"')
self.failUnlessReallyEqual(quote_path([u"foo", u"\nbar"], quotemarks=True), '"foo/\\x0abar"')
self.failUnlessReallyEqual(quote_path([u"foo", u"\nbar"], quotemarks=False), '"foo/\\x0abar"')
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo"), win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'")
)
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True), win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'")
)
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False), win32_other("C:\\foo", "\\\\?\\C:\\foo")
)
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"), win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'")
)
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True),
win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"),
)
self.failUnlessReallyEqual(
quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=False),
win32_other("\\\\foo\\bar", "\\\\?\\UNC\\foo\\bar"),
)
def create_introducer(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "introducer")
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.close()
print >>out, "Introducer created in %s" % quote_local_unicode_path(basedir)
return 0
def opt_exclude_from(self, filepath):
"""Ignore file matching glob patterns listed in file, one per
line. The file is assumed to be in the argv encoding."""
abs_filepath = argv_to_abspath(filepath)
try:
exclude_file = open(abs_filepath)
except:
raise BackupConfigurationError('Error opening exclude file %s.' % quote_local_unicode_path(abs_filepath))
try:
for line in exclude_file:
self.opt_exclude(line)
finally:
exclude_file.close()
def opt_exclude_from(self, filepath):
"""Ignore file matching glob patterns listed in file, one per
line. The file is assumed to be in the argv encoding."""
abs_filepath = argv_to_abspath(filepath)
try:
exclude_file = file(abs_filepath)
except:
raise BackupConfigurationError('Error opening exclude file %s.' % quote_local_unicode_path(abs_filepath))
try:
for line in exclude_file:
self.opt_exclude(line)
finally:
exclude_file.close()
def create_introducer(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print("The base directory %s is not empty." % quote_local_unicode_path(basedir), file=err)
print("To avoid clobbering anything, I am going to quit now.", file=err)
print("Please use a different directory, or empty this one.", file=err)
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "introducer")
fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
print("Introducer created in %s" % quote_local_unicode_path(basedir), file=out)
defer.returnValue(0)
def __init__(self, basedir=u".", verbose=True):
self.verbose = verbose
StatsGatherer.__init__(self, basedir)
self.picklefile = os.path.join(basedir, "stats.pickle")
if os.path.exists(self.picklefile):
f = open(self.picklefile, 'rb')
try:
self.gathered_stats = pickle.load(f)
except Exception:
print ("Error while attempting to load pickle file %s.\n"
"You may need to restore this file from a backup, or delete it if no backup is available.\n" %
quote_local_unicode_path(self.picklefile))
raise
f.close()
else:
self.gathered_stats = {}
def __init__(self, basedir=u".", verbose=True):
self.verbose = verbose
StatsGatherer.__init__(self, basedir)
self.jsonfile = os.path.join(basedir, "stats.json")
if os.path.exists(self.jsonfile):
try:
with open(self.jsonfile, 'rb') as f:
self.gathered_stats = json.load(f)
except Exception:
print("Error while attempting to load stats file %s.\n"
"You may need to restore this file from a backup,"
" or delete it if no backup is available.\n" %
quote_local_unicode_path(self.jsonfile))
raise
else:
self.gathered_stats = {}
class _CreateBaseOptions(BasedirOptions):
optParameters = [
# we provide 'create-node'-time options for the most common
# configuration knobs. The rest can be controlled by editing
# tahoe.cfg before node startup.
("nickname", "n", None, "Specify the nickname for this node."),
("introducer", "i", None, "Specify the introducer FURL to use."),
("webport", "p", "tcp:3456:interface=127.0.0.1",
"Specify which TCP port to run the HTTP interface on. Use 'none' to disable."),
("basedir", "C", None, "Specify which Tahoe base directory should be used. This has the same effect as the global --node-directory option. [default: %s]"
% quote_local_unicode_path(_default_nodedir)),
]
# This is overridden in order to ensure we get a "Wrong number of
# arguments." error when more than one argument is given.
def parseArgs(self, basedir=None):
BasedirOptions.parseArgs(self, basedir)
def do_update_db(written_abspath_u):
filecap = item.file_node.get_uri()
last_uploaded_uri = item.metadata.get('last_uploaded_uri', None)
self._log("DOUPDATEDB %r" % written_abspath_u)
last_downloaded_uri = filecap
last_downloaded_timestamp = now
written_pathinfo = get_pathinfo(written_abspath_u)
if not written_pathinfo.exists and not item.metadata.get('deleted', False):
raise Exception("downloaded object %s disappeared" % quote_local_unicode_path(written_abspath_u))
self._db.did_upload_version(
item.relpath_u, item.metadata['version'], last_uploaded_uri,
last_downloaded_uri, last_downloaded_timestamp, written_pathinfo,
)
self._count('objects_downloaded')
item.set_status('success', self._clock.seconds())
return True
def do_update_db(written_abspath_u):
filecap = item.file_node.get_uri()
last_uploaded_uri = item.metadata.get('last_uploaded_uri', None)
self._log("DOUPDATEDB %r" % written_abspath_u)
last_downloaded_uri = filecap
last_downloaded_timestamp = now
written_pathinfo = get_pathinfo(written_abspath_u)
if not written_pathinfo.exists and not item.metadata.get('deleted', False):
raise Exception("downloaded object %s disappeared" % quote_local_unicode_path(written_abspath_u))
self._db.did_upload_version(
item.relpath_u, item.metadata['version'], last_uploaded_uri,
last_downloaded_uri, last_downloaded_timestamp, written_pathinfo,
)
self._count('objects_downloaded')
item.set_status('success', self._clock.seconds())
return True
def daemonize(config):
"""
Runs the 'tahoe daemonize' command.
Sets up the IService instance corresponding to the type of node
that's starting and uses Twisted's twistd runner to disconnect our
process from the terminal.
"""
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >> out, "daemonizing in {}".format(quoted_basedir)
if not os.path.isdir(basedir):
print >> err, "%s does not look like a directory at all" % quoted_basedir
return 1
nodetype = identify_node_type(basedir)
if not nodetype:
print >> err, "%s is not a recognizable node directory" % quoted_basedir
return 1
# Now prepare to turn into a twistd process. This os.chdir is the point
# of no return.
os.chdir(basedir)
twistd_args = []
if (nodetype in (u"client", u"introducer")
and "--nodaemon" not in config.twistd_args
and "--syslog" not in config.twistd_args
and "--logfile" not in config.twistd_args):
fileutil.make_dirs(os.path.join(basedir, u"logs"))
twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")])
twistd_args.extend(config.twistd_args)
twistd_args.append(
"DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin
twistd_config = MyTwistdConfig()
try:
twistd_config.parseOptions(twistd_args)
except usage.error, ue:
# these arguments were unsuitable for 'twistd'
print >> err, config
print >> err, "tahoe %s: usage error from twistd: %s\n" % (
config.subcommand_name, ue)
return 1
def leave(options):
from ConfigParser import SafeConfigParser
existing_folders = load_magic_folders(options["node-directory"])
if not existing_folders:
print("No magic-folders at all", file=options.stderr)
return 1
if options["name"] not in existing_folders:
print("No such magic-folder '{}'".format(options["name"]),
file=options.stderr)
return 1
privdir = os.path.join(options["node-directory"], u"private")
db_fname = os.path.join(privdir,
u"magicfolder_{}.sqlite".format(options["name"]))
# delete from YAML file and re-write it
del existing_folders[options["name"]]
save_magic_folders(options["node-directory"], existing_folders)
# delete the database file
try:
fileutil.remove(db_fname)
except Exception as e:
print(
"Warning: unable to remove %s due to %s: %s" %
(quote_local_unicode_path(db_fname), e.__class__.__name__, str(e)),
file=options.stderr)
# if this was the last magic-folder, disable them entirely
if not existing_folders:
parser = SafeConfigParser()
parser.read(os.path.join(options["node-directory"], u"tahoe.cfg"))
parser.remove_section("magic_folder")
with open(os.path.join(options["node-directory"], u"tahoe.cfg"),
"w") as f:
parser.write(f)
return 0
def daemonize(config):
"""
Runs the 'tahoe daemonize' command.
Sets up the IService instance corresponding to the type of node
that's starting and uses Twisted's twistd runner to disconnect our
process from the terminal.
"""
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >>out, "daemonizing in {}".format(quoted_basedir)
if not os.path.isdir(basedir):
print >>err, "%s does not look like a directory at all" % quoted_basedir
return 1
nodetype = identify_node_type(basedir)
if not nodetype:
print >>err, "%s is not a recognizable node directory" % quoted_basedir
return 1
# Now prepare to turn into a twistd process. This os.chdir is the point
# of no return.
os.chdir(basedir)
twistd_args = []
if (nodetype in (u"client", u"introducer")
and "--nodaemon" not in config.twistd_args
and "--syslog" not in config.twistd_args
and "--logfile" not in config.twistd_args):
fileutil.make_dirs(os.path.join(basedir, u"logs"))
twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")])
twistd_args.extend(config.twistd_args)
twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin
twistd_config = MyTwistdConfig()
try:
twistd_config.parseOptions(twistd_args)
except usage.error, ue:
# these arguments were unsuitable for 'twistd'
print >>err, config
print >>err, "tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue)
return 1
def leave(options):
from ConfigParser import SafeConfigParser
dmd_cap_file = os.path.join(options["node-directory"], u"private", u"magic_folder_dircap")
collective_readcap_file = os.path.join(options["node-directory"], u"private", u"collective_dircap")
magic_folder_db_file = os.path.join(options["node-directory"], u"private", u"magicfolderdb.sqlite")
parser = SafeConfigParser()
parser.read(os.path.join(options["node-directory"], u"tahoe.cfg"))
parser.remove_section("magic_folder")
f = open(os.path.join(options["node-directory"], u"tahoe.cfg"), "w")
parser.write(f)
f.close()
for f in [dmd_cap_file, collective_readcap_file, magic_folder_db_file]:
try:
fileutil.remove(f)
except Exception as e:
print >>options.stderr, ("Warning: unable to remove %s due to %s: %s"
% (quote_local_unicode_path(f), e.__class__.__name__, str(e)))
# if this doesn't return 0, then the CLI stuff fails
return 0
def stop(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >> out, "STOPPING", quoted_basedir
pidfile = os.path.join(basedir, u"twistd.pid")
if not os.path.exists(pidfile):
print >> err, "%s does not look like a running node directory (no twistd.pid)" % quoted_basedir
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return COULD_NOT_STOP
with open(pidfile, "r") as f:
pid = f.read()
try:
pid = int(pid)
except ValueError:
# The error message below mimics a Twisted error message, which is
# displayed when starting a node with an invalid pidfile.
print >> err, "Pidfile %s contains non-numeric value" % pidfile
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return 2
# kill it hard (SIGKILL), delete the twistd.pid file, then wait for the
# process itself to go away. If it hasn't gone away after 20 seconds, warn
# the user but keep waiting until they give up.
try:
os.kill(pid, signal.SIGKILL)
except OSError, oserr:
if oserr.errno == 3:
print oserr.strerror
# the process didn't exist, so wipe the pid file
os.remove(pidfile)
return COULD_NOT_STOP
else:
raise
def leave(options):
from ConfigParser import SafeConfigParser
existing_folders = load_magic_folders(options["node-directory"])
if not existing_folders:
print("No magic-folders at all", file=options.stderr)
return 1
if options["name"] not in existing_folders:
print("No such magic-folder '{}'".format(options["name"]), file=options.stderr)
return 1
privdir = os.path.join(options["node-directory"], u"private")
db_fname = os.path.join(privdir, u"magicfolder_{}.sqlite".format(options["name"]))
# delete from YAML file and re-write it
del existing_folders[options["name"]]
save_magic_folders(options["node-directory"], existing_folders)
# delete the database file
try:
fileutil.remove(db_fname)
except Exception as e:
print("Warning: unable to remove %s due to %s: %s"
% (quote_local_unicode_path(db_fname), e.__class__.__name__, str(e)), file=options.stderr)
# if this was the last magic-folder, disable them entirely
if not existing_folders:
parser = SafeConfigParser()
parser.read(os.path.join(options["node-directory"], u"tahoe.cfg"))
parser.remove_section("magic_folder")
with open(os.path.join(options["node-directory"], u"tahoe.cfg"), "w") as f:
parser.write(f)
return 0
def find_shares(options):
"""Given a storage index and a list of node directories, emit a list of
all matching shares to stdout, one per line. For example:
find-shares.py 44kai1tui348689nrw8fjegc8c ~/testnet/node-*
gives:
/home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/5
/home/warner/testnet/node-1/storage/shares/44k/44kai1tui348689nrw8fjegc8c/9
/home/warner/testnet/node-2/storage/shares/44k/44kai1tui348689nrw8fjegc8c/2
"""
from allmydata.storage.server import si_a2b, storage_index_to_dir
from allmydata.util.encodingutil import listdir_unicode, quote_local_unicode_path
out = options.stdout
sharedir = storage_index_to_dir(si_a2b(options.si_s))
for d in options.nodedirs:
d = os.path.join(d, "storage", "shares", sharedir)
if os.path.exists(d):
for shnum in listdir_unicode(d):
print >>out, quote_local_unicode_path(os.path.join(d, shnum), quotemarks=False)
return 0
def test_quote_path(self):
self.failUnlessReallyEqual(quote_path([u'foo', u'bar']), "'foo/bar'")
self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=True), "'foo/bar'")
self.failUnlessReallyEqual(quote_path([u'foo', u'bar'], quotemarks=False), "foo/bar")
self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar']), '"foo/\\x0abar"')
self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=True), '"foo/\\x0abar"')
self.failUnlessReallyEqual(quote_path([u'foo', u'\nbar'], quotemarks=False), '"foo/\\x0abar"')
def win32_other(win32, other):
return win32 if sys.platform == "win32" else other
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo"),
win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=True),
win32_other("'C:\\foo'", "'\\\\?\\C:\\foo'"))
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\C:\\foo", quotemarks=False),
win32_other("C:\\foo", "\\\\?\\C:\\foo"))
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar"),
win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=True),
win32_other("'\\\\foo\\bar'", "'\\\\?\\UNC\\foo\\bar'"))
self.failUnlessReallyEqual(quote_local_unicode_path(u"\\\\?\\UNC\\foo\\bar", quotemarks=False),
win32_other("\\\\foo\\bar", "\\\\?\\UNC\\foo\\bar"))
def create_node(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
defer.returnValue(-1)
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
# if we're doing magic-wormhole stuff, do it now
if config['join'] is not None:
try:
remote_config = yield _get_config_via_wormhole(config)
except RuntimeError as e:
print >>err, str(e)
defer.returnValue(1)
# configuration we'll allow the inviter to set
whitelist = [
'shares-happy', 'shares-needed', 'shares-total',
'introducer', 'nickname',
]
sensitive_keys = ['introducer']
print >>out, "Encoding: {shares-needed} of {shares-total} shares, on at least {shares-happy} servers".format(**remote_config)
print >>out, "Overriding the following config:"
for k in whitelist:
v = remote_config.get(k, None)
if v is not None:
# we're faking usually argv-supplied options :/
if isinstance(v, unicode):
v = v.encode(get_io_encoding())
config[k] = v
if k not in sensitive_keys:
if k not in ['shares-happy', 'shares-total', 'shares-needed']:
print >>out, " {}: {}".format(k, v)
else:
print >>out, " {}: [sensitive data; see tahoe.cfg]".format(k)
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
yield write_node_config(c, config)
write_client_config(c, config)
print >>out, "Node created in %s" % quote_local_unicode_path(basedir)
tahoe_cfg = quote_local_unicode_path(os.path.join(basedir, "tahoe.cfg"))
if not config.get("introducer", ""):
print >>out, " Please set [client]introducer.furl= in %s!" % tahoe_cfg
print >>out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >>out, " Please set [node]nickname= in %s" % tahoe_cfg
defer.returnValue(0)
def stop(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print("STOPPING", quoted_basedir, file=out)
pidfile = get_pidfile(basedir)
pid = get_pid_from_pidfile(pidfile)
if pid is None:
print(
"%s does not look like a running node directory (no twistd.pid)" %
quoted_basedir,
file=err)
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return COULD_NOT_STOP
elif pid == -1:
print("%s contains an invalid PID file" % basedir, file=err)
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return COULD_NOT_STOP
# kill it hard (SIGKILL), delete the twistd.pid file, then wait for the
# process itself to go away. If it hasn't gone away after 20 seconds, warn
# the user but keep waiting until they give up.
try:
os.kill(pid, signal.SIGKILL)
except OSError as oserr:
if oserr.errno == 3:
print(oserr.strerror)
# the process didn't exist, so wipe the pid file
os.remove(pidfile)
return COULD_NOT_STOP
else:
raise
try:
os.remove(pidfile)
except EnvironmentError:
pass
start = time.time()
time.sleep(0.1)
wait = 40
first_time = True
while True:
# poll once per second until we see the process is no longer running
try:
os.kill(pid, 0)
except OSError:
print("process %d is dead" % pid, file=out)
return
wait -= 1
if wait < 0:
if first_time:
print("It looks like pid %d is still running "
"after %d seconds" % (pid, (time.time() - start)),
file=err)
print("I will keep watching it until you interrupt me.",
file=err)
wait = 10
first_time = False
else:
print("pid %d still running after %d seconds" % \
(pid, (time.time() - start)), file=err)
wait = 10
time.sleep(1)
def create_node(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.write("[client]\n")
c.write("# Which services should this client connect to?\n")
c.write("introducer.furl = %s\n" % config.get("introducer", ""))
c.write("helper.furl =\n")
c.write("#key_generator.furl =\n")
c.write("#stats_gatherer.furl =\n")
c.write("\n")
c.write("# What encoding parameters should this client use for uploads?\n")
c.write("#shares.needed = 3\n")
c.write("#shares.happy = 7\n")
c.write("#shares.total = 10\n")
c.write("\n")
boolstr = {True:"true", False:"false"}
c.write("[storage]\n")
c.write("# Shall this node provide storage service?\n")
storage_enabled = not config.get("no-storage", None)
c.write("enabled = %s\n" % boolstr[storage_enabled])
c.write("#readonly =\n")
c.write("reserved_space = 1G\n")
c.write("#expire.enabled =\n")
c.write("#expire.mode =\n")
c.write("\n")
c.write("[helper]\n")
c.write("# Shall this node run a helper service that clients can use?\n")
c.write("enabled = false\n")
c.write("\n")
c.write("[drop_upload]\n")
c.write("# Shall this node automatically upload files created or modified in a local directory?\n")
c.write("enabled = false\n")
c.write("# To specify the target of uploads, a mutable directory writecap URI must be placed\n"
"# in 'private/drop_upload_dircap'.\n")
c.write("local.directory = ~/drop_upload\n")
c.write("\n")
c.close()
from allmydata.util import fileutil
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
print >>out, "Node created in %s" % quote_local_unicode_path(basedir)
if not config.get("introducer", ""):
print >>out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >>out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >>out, " Please set [node]nickname= in tahoe.cfg"
return 0
def daemonize(config):
"""
Runs the 'tahoe daemonize' command.
Sets up the IService instance corresponding to the type of node
that's starting and uses Twisted's twistd runner to disconnect our
process from the terminal.
"""
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print("daemonizing in {}".format(quoted_basedir), file=out)
if not os.path.isdir(basedir):
print("%s does not look like a directory at all" % quoted_basedir, file=err)
return 1
nodetype = identify_node_type(basedir)
if not nodetype:
print("%s is not a recognizable node directory" % quoted_basedir, file=err)
return 1
# Now prepare to turn into a twistd process. This os.chdir is the point
# of no return.
os.chdir(basedir)
twistd_args = []
if (nodetype in (u"client", u"introducer")
and "--nodaemon" not in config.twistd_args
and "--syslog" not in config.twistd_args
and "--logfile" not in config.twistd_args):
fileutil.make_dirs(os.path.join(basedir, u"logs"))
twistd_args.extend(["--logfile", os.path.join("logs", "twistd.log")])
twistd_args.extend(config.twistd_args)
twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin
twistd_config = MyTwistdConfig()
try:
twistd_config.parseOptions(twistd_args)
except usage.error as ue:
# these arguments were unsuitable for 'twistd'
print(config, file=err)
print("tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue), file=err)
return 1
twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)}
# handle invalid PID file (twistd might not start otherwise)
pidfile = get_pidfile(basedir)
if get_pid_from_pidfile(pidfile) == -1:
print("found invalid PID file in %s - deleting it" % basedir, file=err)
os.remove(pidfile)
# On Unix-like platforms:
# Unless --nodaemon was provided, the twistd.runApp() below spawns off a
# child process, and the parent calls os._exit(0), so there's no way for
# us to get control afterwards, even with 'except SystemExit'. If
# application setup fails (e.g. ImportError), runApp() will raise an
# exception.
#
# So if we wanted to do anything with the running child, we'd have two
# options:
#
# * fork first, and have our child wait for the runApp() child to get
# running. (note: just fork(). This is easier than fork+exec, since we
# don't have to get PATH and PYTHONPATH set up, since we're not
# starting a *different* process, just cloning a new instance of the
# current process)
# * or have the user run a separate command some time after this one
# exits.
#
# For Tahoe, we don't need to do anything with the child, so we can just
# let it exit.
#
# On Windows:
# twistd does not fork; it just runs in the current process whether or not
# --nodaemon is specified. (As on Unix, --nodaemon does have the side effect
# of causing us to log to stdout/stderr.)
if "--nodaemon" in twistd_args or sys.platform == "win32":
verb = "running"
else:
verb = "starting"
print("%s node in %s" % (verb, quoted_basedir), file=out)
twistd.runApp(twistd_config)
# we should only reach here if --nodaemon or equivalent was used
return 0
opt_eliot_destination,
opt_help_eliot_destinations,
eliot_logging_service,
)
_default_nodedir = get_default_nodedir()
NODEDIR_HELP = ("Specify which Tahoe node directory should be used. The "
"directory should either contain a full Tahoe node, or a "
"file named node.url that points to some other Tahoe node. "
"It should also contain a file named '"
+ os.path.join('private', 'aliases') +
"' which contains the mapping from alias name to root "
"dirnode URI.")
if _default_nodedir:
NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]"
# XXX all this 'dispatch' stuff needs to be unified + fixed up
_control_node_dispatch = {
"daemonize": tahoe_daemonize.daemonize,
"start": tahoe_start.start,
"run": tahoe_run.run,
"stop": tahoe_stop.stop,
"restart": tahoe_restart.restart,
}
process_control_commands = [
["daemonize", None, tahoe_daemonize.DaemonizeOptions, "run a node in the background"],
["start", None, tahoe_start.StartOptions, "start a node in the background and confirm it started"],
["run", None, tahoe_run.RunOptions, "run a node without daemonizing"],
opt_eliot_destination,
opt_help_eliot_destinations,
eliot_logging_service,
)
_default_nodedir = get_default_nodedir()
NODEDIR_HELP = ("Specify which Tahoe node directory should be used. The "
"directory should either contain a full Tahoe node, or a "
"file named node.url that points to some other Tahoe node. "
"It should also contain a file named '" +
os.path.join('private', 'aliases') +
"' which contains the mapping from alias name to root "
"dirnode URI.")
if _default_nodedir:
NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(
_default_nodedir) + "]"
# XXX all this 'dispatch' stuff needs to be unified + fixed up
_control_node_dispatch = {
"daemonize": tahoe_daemonize.daemonize,
"start": tahoe_start.start,
"run": tahoe_run.run,
"stop": tahoe_stop.stop,
"restart": tahoe_restart.restart,
}
process_control_commands = [
["run", None, tahoe_run.RunOptions, "run a node without daemonizing"],
[
"daemonize", None, tahoe_daemonize.DaemonizeOptions,
"(deprecated) run a node in the background"
def __init__(self, path, isdir=False):
self._path = path
self._quoted_path = quote_local_unicode_path(path)
self._isdir = isdir
def __init__(self, absname):
quoted = quote_local_unicode_path(absname)
TahoeError.__init__(self, "source '%s' is neither a file nor a directory, I can't handle it" % quoted)
def start(config):
"""
Start a tahoe node (daemonize it and confirm startup)
We run 'tahoe daemonize' with all the options given to 'tahoe
start' and then watch the log files for the correct text to appear
(e.g. "introducer started"). If that doesn't happen within a few
seconds, an error is printed along with all collected logs.
"""
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print >>out, "STARTING", quoted_basedir
if not os.path.isdir(basedir):
print >>err, "%s does not look like a directory at all" % quoted_basedir
return 1
nodetype = identify_node_type(basedir)
if not nodetype:
print >>err, "%s is not a recognizable node directory" % quoted_basedir
return 1
# "tahoe start" attempts to monitor the logs for successful
# startup -- but we can't always do that.
can_monitor_logs = False
if (nodetype in (u"client", u"introducer")
and "--nodaemon" not in config.twistd_args
and "--syslog" not in config.twistd_args
and "--logfile" not in config.twistd_args):
can_monitor_logs = True
if "--help" in config.twistd_args:
return 0
if not can_monitor_logs:
print >>out, "Custom logging options; can't monitor logs for proper startup messages"
return 1
# before we spawn tahoe, we check if "the log file" exists or not,
# and if so remember how big it is -- essentially, we're doing
# "tail -f" to see what "this" incarnation of "tahoe daemonize"
# spews forth.
starting_offset = 0
log_fname = join(basedir, 'logs', 'twistd.log')
if exists(log_fname):
with open(log_fname, 'r') as f:
f.seek(0, 2)
starting_offset = f.tell()
# spawn tahoe. Note that since this daemonizes, it should return
# "pretty fast" and with a zero return-code, or else something
# Very Bad has happened.
try:
args = [sys.executable] if not getattr(sys, 'frozen', False) else []
for i, arg in enumerate(sys.argv):
if arg in ['start', 'restart']:
args.append('daemonize')
else:
args.append(arg)
subprocess.check_call(args)
except subprocess.CalledProcessError as e:
return e.returncode
# now, we have to determine if tahoe has actually started up
# successfully or not. so, we start sucking up log files and
# looking for "the magic string", which depends on the node type.
magic_string = u'{} running'.format(nodetype)
with io.open(log_fname, 'r') as f:
f.seek(starting_offset)
collected = u''
overall_start = time.time()
while time.time() - overall_start < 60:
this_start = time.time()
while time.time() - this_start < 5:
collected += f.read()
if magic_string in collected:
if not config.parent['quiet']:
print >>out, "Node has started successfully"
return 0
if 'Traceback ' in collected:
print >>err, "Error starting node; see '{}' for more:\n\n{}".format(
log_fname,
collected,
)
return 1
time.sleep(0.1)
print >>out, "Still waiting up to {}s for node startup".format(
60 - int(time.time() - overall_start)
)
print >>out, "Something has gone wrong starting the node."
print >>out, "Logs are available in '{}'".format(log_fname)
print >>out, "Collected for this run:"
print >>out, collected
return 1
def __init__(self, path, isdir):
self._path = path
self._quoted_path = quote_local_unicode_path(path)
self._isdir = isdir
def stop(config):
out = config.stdout
err = config.stderr
basedir = config['basedir']
quoted_basedir = quote_local_unicode_path(basedir)
print("STOPPING", quoted_basedir, file=out)
pidfile = get_pidfile(basedir)
pid = get_pid_from_pidfile(pidfile)
if pid is None:
print("%s does not look like a running node directory (no twistd.pid)" % quoted_basedir, file=err)
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return 2
elif pid == -1:
print("%s contains an invalid PID file" % basedir, file=err)
# we define rc=2 to mean "nothing is running, but it wasn't me who
# stopped it"
return 2
# kill it hard (SIGKILL), delete the twistd.pid file, then wait for the
# process itself to go away. If it hasn't gone away after 20 seconds, warn
# the user but keep waiting until they give up.
try:
os.kill(pid, signal.SIGKILL)
except OSError as oserr:
if oserr.errno == 3:
print(oserr.strerror)
# the process didn't exist, so wipe the pid file
os.remove(pidfile)
return COULD_NOT_STOP
else:
raise
try:
os.remove(pidfile)
except EnvironmentError:
pass
start = time.time()
time.sleep(0.1)
wait = 40
first_time = True
while True:
# poll once per second until we see the process is no longer running
try:
os.kill(pid, 0)
except OSError:
print("process %d is dead" % pid, file=out)
return
wait -= 1
if wait < 0:
if first_time:
print("It looks like pid %d is still running "
"after %d seconds" % (pid,
(time.time() - start)), file=err)
print("I will keep watching it until you interrupt me.", file=err)
wait = 10
first_time = False
else:
print("pid %d still running after %d seconds" % \
(pid, (time.time() - start)), file=err)
wait = 10
time.sleep(1)
def create_node(config, out=sys.stdout, err=sys.stderr):
basedir = config['basedir']
# This should always be called with an absolute Unicode basedir.
precondition(isinstance(basedir, unicode), basedir)
if os.path.exists(basedir):
if listdir_unicode(basedir):
print >>err, "The base directory %s is not empty." % quote_local_unicode_path(basedir)
print >>err, "To avoid clobbering anything, I am going to quit now."
print >>err, "Please use a different directory, or empty this one."
return -1
# we're willing to use an empty directory
else:
os.mkdir(basedir)
write_tac(basedir, "client")
c = open(os.path.join(basedir, "tahoe.cfg"), "w")
write_node_config(c, config)
c.write("[client]\n")
c.write("# Which services should this client connect to?\n")
c.write("introducer.furl = %s\n" % config.get("introducer", ""))
c.write("helper.furl =\n")
c.write("#stats_gatherer.furl =\n")
c.write("\n")
c.write("# Encoding parameters this client will use for newly-uploaded files\n")
c.write("# This can be changed at any time: the encoding is saved in\n")
c.write("# each filecap, and we can download old files with any encoding\n")
c.write("# settings\n")
c.write("#shares.needed = 3\n")
c.write("#shares.happy = 7\n")
c.write("#shares.total = 10\n")
c.write("\n")
boolstr = {True:"true", False:"false"}
c.write("[storage]\n")
c.write("# Shall this node provide storage service?\n")
storage_enabled = not config.get("no-storage", None)
c.write("enabled = %s\n" % boolstr[storage_enabled])
c.write("#readonly =\n")
c.write("reserved_space = 1G\n")
c.write("#expire.enabled =\n")
c.write("#expire.mode =\n")
c.write("\n")
c.write("[helper]\n")
c.write("# Shall this node run a helper service that clients can use?\n")
c.write("enabled = false\n")
c.write("\n")
c.close()
from allmydata.util import fileutil
fileutil.make_dirs(os.path.join(basedir, "private"), 0700)
print >>out, "Node created in %s" % quote_local_unicode_path(basedir)
if not config.get("introducer", ""):
print >>out, " Please set [client]introducer.furl= in tahoe.cfg!"
print >>out, " The node cannot connect to a grid without it."
if not config.get("nickname", ""):
print >>out, " Please set [node]nickname= in tahoe.cfg"
return 0
