def setup_logging(self):
# we replace the formatTime() method of the log observer that
# twistd set up for us, with a method that uses our preferred
# timestamp format.
for o in twlog.theLogPublisher.observers:
# o might be a FileLogObserver's .emit method
if type(o) is type(self.setup_logging): # bound method
ob = o.im_self
if isinstance(ob, twlog.FileLogObserver):
newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__)
ob.formatTime = newmeth
# TODO: twisted >2.5.0 offers maxRotatedFiles=50
lgfurl_file = os.path.join(self.basedir, "private", "logport.furl").encode(get_filesystem_encoding())
if os.path.exists(lgfurl_file):
os.remove(lgfurl_file)
self.log_tub.setOption("logport-furlfile", lgfurl_file)
lgfurl = self.get_config("node", "log_gatherer.furl", "")
if lgfurl:
# this is in addition to the contents of log-gatherer-furlfile
self.log_tub.setOption("log-gatherer-furl", lgfurl)
self.log_tub.setOption("log-gatherer-furlfile",
os.path.join(self.basedir, "log_gatherer.furl"))
incident_dir = os.path.join(self.basedir, "logs", "incidents")
foolscap.logging.log.setLogDir(incident_dir.encode(get_filesystem_encoding()))
twlog.msg("Foolscap logging initialized")
twlog.msg("Note to developers: twistd.log does not receive very much.")
twlog.msg("Use 'flogtool tail -c NODEDIR/private/logport.furl' instead")
twlog.msg("and read docs/logging.rst")
def setup_logging(self):
# we replace the formatTime() method of the log observer that
# twistd set up for us, with a method that uses our preferred
# timestamp format.
for o in twlog.theLogPublisher.observers:
# o might be a FileLogObserver's .emit method
if type(o) is type(self.setup_logging): # bound method
ob = o.im_self
if isinstance(ob, twlog.FileLogObserver):
newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__)
ob.formatTime = newmeth
# TODO: twisted >2.5.0 offers maxRotatedFiles=50
lgfurl_file = os.path.join(self.basedir, "private", "logport.furl").encode(get_filesystem_encoding())
self.tub.setOption("logport-furlfile", lgfurl_file)
lgfurl = self.get_config("node", "log_gatherer.furl", "")
if lgfurl:
# this is in addition to the contents of log-gatherer-furlfile
self.tub.setOption("log-gatherer-furl", lgfurl)
self.tub.setOption("log-gatherer-furlfile",
os.path.join(self.basedir, "log_gatherer.furl"))
self.tub.setOption("bridge-twisted-logs", True)
incident_dir = os.path.join(self.basedir, "logs", "incidents")
# this doesn't quite work yet: unit tests fail
foolscap.logging.log.setLogDir(incident_dir.encode(get_filesystem_encoding()))
def setup_logging(self):
# we replace the formatTime() method of the log observer that
# twistd set up for us, with a method that uses our preferred
# timestamp format.
for o in twlog.theLogPublisher.observers:
# o might be a FileLogObserver's .emit method
if type(o) is type(self.setup_logging): # bound method
ob = o.im_self
if isinstance(ob, twlog.FileLogObserver):
newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob,
ob.__class__)
ob.formatTime = newmeth
# TODO: twisted >2.5.0 offers maxRotatedFiles=50
lgfurl_file = os.path.join(self.basedir, "private",
"logport.furl").encode(
get_filesystem_encoding())
if os.path.exists(lgfurl_file):
os.remove(lgfurl_file)
self.log_tub.setOption("logport-furlfile", lgfurl_file)
lgfurl = self.config.get_config("node", "log_gatherer.furl", "")
if lgfurl:
# this is in addition to the contents of log-gatherer-furlfile
self.log_tub.setOption("log-gatherer-furl", lgfurl)
self.log_tub.setOption("log-gatherer-furlfile",
os.path.join(self.basedir, "log_gatherer.furl"))
incident_dir = os.path.join(self.basedir, "logs", "incidents")
foolscap.logging.log.setLogDir(
incident_dir.encode(get_filesystem_encoding()))
twlog.msg("Foolscap logging initialized")
twlog.msg("Note to developers: twistd.log does not receive very much.")
twlog.msg(
"Use 'flogtool tail -c NODEDIR/private/logport.furl' instead")
twlog.msg("and read docs/logging.rst")
def init_introducer(self):
introducerservice = IntroducerService(self.basedir)
self.add_service(introducerservice)
old_public_fn = os.path.join(self.basedir, "introducer.furl").encode(
get_filesystem_encoding())
private_fn = os.path.join(self.basedir, "private",
"introducer.furl").encode(
get_filesystem_encoding())
if os.path.exists(old_public_fn):
if os.path.exists(private_fn):
msg = """This directory (%s) contains both an old public
'introducer.furl' file, and a new-style
'private/introducer.furl', so I cannot safely remove the old
one. Please make sure your desired FURL is in
private/introducer.furl, and remove the public file. If this
causes your Introducer's FURL to change, you need to inform
all grid members so they can update their tahoe.cfg.
"""
raise FurlFileConflictError(textwrap.dedent(msg))
os.rename(old_public_fn, private_fn)
d = self.when_tub_ready()
def _publish(res):
furl = self.tub.registerReference(introducerservice,
furlFile=private_fn)
self.log(" introducer is at %s" % furl, umid="qF2L9A")
self.introducer_url = furl # for tests
d.addCallback(_publish)
d.addErrback(log.err,
facility="tahoe.init",
level=log.BAD,
umid="UaNs9A")
def init_introducer(self):
introducerservice = IntroducerService(self.basedir)
self.add_service(introducerservice)
old_public_fn = os.path.join(self.basedir, "introducer.furl").encode(get_filesystem_encoding())
private_fn = os.path.join(self.basedir, "private", "introducer.furl").encode(get_filesystem_encoding())
if os.path.exists(old_public_fn):
if os.path.exists(private_fn):
msg = """This directory (%s) contains both an old public
'introducer.furl' file, and a new-style
'private/introducer.furl', so I cannot safely remove the old
one. Please make sure your desired FURL is in
private/introducer.furl, and remove the public file. If this
causes your Introducer's FURL to change, you need to inform
all grid members so they can update their tahoe.cfg.
"""
raise FurlFileConflictError(textwrap.dedent(msg))
os.rename(old_public_fn, private_fn)
d = self.when_tub_ready()
def _publish(res):
furl = self.tub.registerReference(introducerservice,
furlFile=private_fn)
self.log(" introducer is at %s" % furl, umid="qF2L9A")
self.introducer_url = furl # for tests
d.addCallback(_publish)
d.addErrback(log.err, facility="tahoe.init",
level=log.BAD, umid="UaNs9A")
def skip_if_cannot_represent_filename(self, u):
enc = get_filesystem_encoding()
if not unicode_platform():
try:
u.encode(enc)
except UnicodeEncodeError:
raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
def init_storage(self):
# should we run a storage server (and publish it for others to use)?
if not self.get_config("storage", "enabled", True, boolean=True):
return
readonly = self.get_config("storage", "readonly", False, boolean=True)
storedir = os.path.join(self.basedir, self.STOREDIR)
data = self.get_config("storage", "reserved_space", None)
try:
reserved = parse_abbreviated_size(data)
except ValueError:
log.msg("[storage]reserved_space= contains unparseable value %s"
% data)
raise
if reserved is None:
reserved = 0
discard = self.get_config("storage", "debug_discard", False,
boolean=True)
expire = self.get_config("storage", "expire.enabled", False, boolean=True)
if expire:
mode = self.get_config("storage", "expire.mode") # require a mode
else:
mode = self.get_config("storage", "expire.mode", "age")
o_l_d = self.get_config("storage", "expire.override_lease_duration", None)
if o_l_d is not None:
o_l_d = parse_duration(o_l_d)
cutoff_date = None
if mode == "cutoff-date":
cutoff_date = self.get_config("storage", "expire.cutoff_date")
cutoff_date = parse_date(cutoff_date)
sharetypes = []
if self.get_config("storage", "expire.immutable", True, boolean=True):
sharetypes.append("immutable")
if self.get_config("storage", "expire.mutable", True, boolean=True):
sharetypes.append("mutable")
expiration_sharetypes = tuple(sharetypes)
ss = StorageServer(storedir, self.nodeid,
reserved_space=reserved,
discard_storage=discard,
readonly_storage=readonly,
stats_provider=self.stats_provider,
expiration_enabled=expire,
expiration_mode=mode,
expiration_override_lease_duration=o_l_d,
expiration_cutoff_date=cutoff_date,
expiration_sharetypes=expiration_sharetypes)
self.add_service(ss)
furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding())
furl = self.tub.registerReference(ss, furlFile=furl_file)
ann = {"anonymous-storage-FURL": furl,
"permutation-seed-base32": self._init_permutation_seed(ss),
}
self.introducer_client.publish("storage", ann, self._node_key)
def init_storage(self):
# should we run a storage server (and publish it for others to use)?
if not self.get_config("storage", "enabled", True, boolean=True):
return
readonly = self.get_config("storage", "readonly", False, boolean=True)
storedir = os.path.join(self.basedir, self.STOREDIR)
data = self.get_config("storage", "reserved_space", None)
try:
reserved = parse_abbreviated_size(data)
except ValueError:
log.msg("[storage]reserved_space= contains unparseable value %s"
% data)
raise
if reserved is None:
reserved = 0
discard = self.get_config("storage", "debug_discard", False,
boolean=True)
expire = self.get_config("storage", "expire.enabled", False, boolean=True)
if expire:
mode = self.get_config("storage", "expire.mode") # require a mode
else:
mode = self.get_config("storage", "expire.mode", "age")
o_l_d = self.get_config("storage", "expire.override_lease_duration", None)
if o_l_d is not None:
o_l_d = parse_duration(o_l_d)
cutoff_date = None
if mode == "cutoff-date":
cutoff_date = self.get_config("storage", "expire.cutoff_date")
cutoff_date = parse_date(cutoff_date)
sharetypes = []
if self.get_config("storage", "expire.immutable", True, boolean=True):
sharetypes.append("immutable")
if self.get_config("storage", "expire.mutable", True, boolean=True):
sharetypes.append("mutable")
expiration_sharetypes = tuple(sharetypes)
ss = StorageServer(storedir, self.nodeid,
reserved_space=reserved,
discard_storage=discard,
readonly_storage=readonly,
stats_provider=self.stats_provider,
expiration_enabled=expire,
expiration_mode=mode,
expiration_override_lease_duration=o_l_d,
expiration_cutoff_date=cutoff_date,
expiration_sharetypes=expiration_sharetypes)
self.add_service(ss)
furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding())
furl = self.tub.registerReference(ss, furlFile=furl_file)
ann = {"anonymous-storage-FURL": furl,
"permutation-seed-base32": self._init_permutation_seed(ss),
}
self.introducer_client.publish("storage", ann, self._node_key)
def _publish(res):
furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding())
furl = self.tub.registerReference(ss, furlFile=furl_file)
ann = {"anonymous-storage-FURL": furl,
"permutation-seed-base32": self._init_permutation_seed(ss),
}
self.introducer_client.publish("storage", ann, self._node_key)
def init_storage(self, announceable_storage_servers):
# should we run a storage server (and publish it for others to use)?
if not storage_enabled(self.config):
return
if not self._is_tub_listening():
raise ValueError("config error: storage is enabled, but tub "
"is not listening ('tub.port=' is empty)")
ss = self.get_anonymous_storage_server()
announcement = {
"permutation-seed-base32": self._init_permutation_seed(ss),
}
if anonymous_storage_enabled(self.config):
furl_file = self.config.get_private_path("storage.furl").encode(
get_filesystem_encoding())
furl = self.tub.registerReference(ss, furlFile=furl_file)
announcement["anonymous-storage-FURL"] = furl
enabled_storage_servers = self._enable_storage_servers(
announceable_storage_servers, )
storage_options = list(storage_server.announcement
for storage_server in enabled_storage_servers)
plugins_announcement = {}
if storage_options:
# Only add the new key if there are any plugins enabled.
plugins_announcement[u"storage-options"] = storage_options
announcement.update(plugins_announcement)
for ic in self.introducer_clients:
ic.publish("storage", announcement, self._node_private_key)
def skip_if_cannot_represent_filename(self, u):
enc = get_filesystem_encoding()
if not unicode_platform():
try:
u.encode(enc)
except UnicodeEncodeError:
raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
def _publish(res):
furl_file = os.path.join(self.basedir, "private",
"storage.furl").encode(
get_filesystem_encoding())
furl = self.tub.registerReference(ss, furlFile=furl_file)
ri_name = RIStorageServer.__remote_name__
self.introducer_client.publish(furl, "storage", ri_name)
def _publish(res):
furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding())
furl = self.tub.registerReference(ss, furlFile=furl_file)
ann = {"anonymous-storage-FURL": furl,
"permutation-seed-base32": self._init_permutation_seed(ss),
}
self.introducer_client.publish("storage", ann, self._server_key)
def test_the_right_code(self):
# running "tahoe" in a subprocess should find the same code that
# holds this test file, else something is weird
test_path = os.path.dirname(
os.path.dirname(os.path.normcase(os.path.realpath(srcfile))))
bintahoe_import_path = yield self.find_import_location()
same = (bintahoe_import_path == test_path)
if not same:
msg = (
"My tests and my 'tahoe' executable are using different paths.\n"
"tahoe: %r\n"
"tests: %r\n"
"( according to the test source filename %r)\n" %
(bintahoe_import_path, test_path, srcfile))
if (not isinstance(rootdir, unicode) and rootdir.decode(
get_filesystem_encoding(), 'replace') != rootdir):
msg += (
"However, this may be a false alarm because the import path\n"
"is not representable in the filesystem encoding.")
raise unittest.SkipTest(msg)
else:
msg += "Please run the tests in a virtualenv that includes both the Tahoe-LAFS library and the 'tahoe' executable."
self.fail(msg)
def __init__(self, client, upload_dircap, local_dir_utf8, inotify=None):
service.MultiService.__init__(self)
try:
local_dir_u = abspath_expanduser_unicode(
local_dir_utf8.decode('utf-8'))
if sys.platform == "win32":
local_dir = local_dir_u
else:
local_dir = local_dir_u.encode(get_filesystem_encoding())
except (UnicodeEncodeError, UnicodeDecodeError):
raise AssertionError(
"The '[drop_upload] local.directory' parameter %s was not valid UTF-8 or "
"could not be represented in the filesystem encoding." %
quote_output(local_dir_utf8))
self._client = client
self._stats_provider = client.stats_provider
self._convergence = client.convergence
self._local_path = FilePath(local_dir)
self.is_upload_ready = False
if inotify is None:
from twisted.internet import inotify
self._inotify = inotify
if not self._local_path.exists():
raise AssertionError(
"The '[drop_upload] local.directory' parameter was %s but there is no directory at that location."
% quote_output(local_dir_u))
if not self._local_path.isdir():
raise AssertionError(
"The '[drop_upload] local.directory' parameter was %s but the thing at that location is not a directory."
% quote_output(local_dir_u))
# TODO: allow a path rather than a cap URI.
self._parent = self._client.create_node_from_uri(upload_dircap)
if not IDirectoryNode.providedBy(self._parent):
raise AssertionError(
"The URI in 'private/drop_upload_dircap' does not refer to a directory."
)
if self._parent.is_unknown() or self._parent.is_readonly():
raise AssertionError(
"The URI in 'private/drop_upload_dircap' is not a writecap to a directory."
)
self._uploaded_callback = lambda ign: None
self._notifier = inotify.INotify()
# We don't watch for IN_CREATE, because that would cause us to read and upload a
# possibly-incomplete file before the application has closed it. There should always
# be an IN_CLOSE_WRITE after an IN_CREATE (I think).
# TODO: what about IN_MOVE_SELF or IN_UNMOUNT?
mask = inotify.IN_CLOSE_WRITE | inotify.IN_MOVED_TO | inotify.IN_ONLYDIR
self._notifier.watch(self._local_path,
mask=mask,
callbacks=[self._notify])
def _add_file(ign):
name = path.basename()
# on Windows the name is already Unicode
if not isinstance(name, unicode):
name = name.decode(get_filesystem_encoding())
u = FileName(path.path, self._convergence)
return self._parent.add_file(name, u)
def unicode_or_fallback(self, unicode_name, fallback_name):
if unicode_platform():
return unicode_name
try:
unicode_name.encode(get_filesystem_encoding())
return unicode_name
except UnicodeEncodeError:
return fallback_name
def _add_file(ign):
name = path.basename()
# on Windows the name is already Unicode
if not isinstance(name, unicode):
name = name.decode(get_filesystem_encoding())
u = FileName(path.path, self._convergence)
return self._parent.add_file(name, u)
def setup_ssh(self):
ssh_port = self.get_config("node", "ssh.port", "")
if ssh_port:
ssh_keyfile = self.get_config("node", "ssh.authorized_keys_file").decode('utf-8')
from allmydata import manhole
m = manhole.AuthorizedKeysManhole(ssh_port, ssh_keyfile.encode(get_filesystem_encoding()))
m.setServiceParent(self)
self.log("AuthorizedKeysManhole listening on %s" % ssh_port)
def unicode_or_fallback(self, unicode_name, fallback_name):
if unicode_platform():
return unicode_name
try:
unicode_name.encode(get_filesystem_encoding())
return unicode_name
except UnicodeEncodeError:
return fallback_name
def setup_ssh(self):
ssh_port = self.get_config("node", "ssh.port", "")
if ssh_port:
ssh_keyfile = self.get_config("node", "ssh.authorized_keys_file").decode('utf-8')
from allmydata import manhole
m = manhole.AuthorizedKeysManhole(ssh_port, ssh_keyfile.encode(get_filesystem_encoding()))
m.setServiceParent(self)
self.log("AuthorizedKeysManhole listening on %s" % ssh_port)
def process(self, event):
event_filepath_u = event.src_path.decode(encodingutil.get_filesystem_encoding())
event_filepath_u = abspath_expanduser_unicode(event_filepath_u, base=self._path)
if event_filepath_u == self._path:
# ignore events for parent directory
return
self._maybe_notify(event_filepath_u, event)
def process(self, event):
event_filepath_u = event.src_path.decode(encodingutil.get_filesystem_encoding())
event_filepath_u = abspath_expanduser_unicode(event_filepath_u, base=self._path)
if event_filepath_u == self._path:
# ignore events for parent directory
return
self._maybe_notify(event_filepath_u, event)
def init_helper(self):
self.helper = Helper(self.config.get_config_path("helper"),
self.storage_broker, self._secret_holder,
self.stats_provider, self.history)
# TODO: this is confusing. BASEDIR/private/helper.furl is created by
# the helper. BASEDIR/helper.furl is consumed by the client who wants
# to use the helper. I like having the filename be the same, since
# that makes 'cp' work smoothly, but the difference between config
# inputs and generated outputs is hard to see.
helper_furlfile = self.config.get_private_path("helper.furl").encode(get_filesystem_encoding())
self.tub.registerReference(self.helper, furlFile=helper_furlfile)
def test_open_unrepresentable(self):
if unicode_platform():
raise unittest.SkipTest("This test is not applicable to platforms that represent filenames as Unicode.")
enc = get_filesystem_encoding()
fn = u'\u2621.txt'
try:
fn.encode(enc)
raise unittest.SkipTest("This test cannot be run unless we know a filename that is not representable.")
except UnicodeEncodeError:
self.failUnlessRaises(UnicodeEncodeError, open, fn, 'wb')
def test_open_unrepresentable(self):
if unicode_platform():
raise unittest.SkipTest("This test is not applicable to platforms that represent filenames as Unicode.")
enc = get_filesystem_encoding()
fn = u'\u2621.txt'
try:
fn.encode(enc)
raise unittest.SkipTest("This test cannot be run unless we know a filename that is not representable.")
except UnicodeEncodeError:
self.failUnlessRaises(UnicodeEncodeError, open, fn, 'wb')
def init_helper(self):
self.helper = Helper(self.config.get_config_path("helper"),
self.storage_broker, self._secret_holder,
self.stats_provider, self.history)
# TODO: this is confusing. BASEDIR/private/helper.furl is created by
# the helper. BASEDIR/helper.furl is consumed by the client who wants
# to use the helper. I like having the filename be the same, since
# that makes 'cp' work smoothly, but the difference between config
# inputs and generated outputs is hard to see.
helper_furlfile = self.config.get_private_path("helper.furl").encode(get_filesystem_encoding())
self.tub.registerReference(self.helper, furlFile=helper_furlfile)
def _publish(self):
self.helper = Helper(os.path.join(self.basedir, "helper"),
self.storage_broker, self._secret_holder,
self.stats_provider, self.history)
# TODO: this is confusing. BASEDIR/private/helper.furl is created
# by the helper. BASEDIR/helper.furl is consumed by the client
# who wants to use the helper. I like having the filename be the
# same, since that makes 'cp' work smoothly, but the difference
# between config inputs and generated outputs is hard to see.
helper_furlfile = os.path.join(self.basedir,
"private", "helper.furl").encode(get_filesystem_encoding())
self.tub.registerReference(self.helper, furlFile=helper_furlfile)
def _publish(self):
self.helper = Helper(os.path.join(self.basedir, "helper"),
self.storage_broker, self._secret_holder,
self.stats_provider, self.history)
# TODO: this is confusing. BASEDIR/private/helper.furl is created
# by the helper. BASEDIR/helper.furl is consumed by the client
# who wants to use the helper. I like having the filename be the
# same, since that makes 'cp' work smoothly, but the difference
# between config inputs and generated outputs is hard to see.
helper_furlfile = os.path.join(self.basedir,
"private", "helper.furl").encode(get_filesystem_encoding())
self.tub.registerReference(self.helper, furlFile=helper_furlfile)
def unicode_or_fallback(self, unicode_name, fallback_name, io_as_well=False):
if not unicode_platform():
try:
unicode_name.encode(get_filesystem_encoding())
except UnicodeEncodeError:
return fallback_name
if io_as_well:
try:
unicode_name.encode(get_io_encoding())
except UnicodeEncodeError:
return fallback_name
return unicode_name
def unicode_or_fallback(self, unicode_name, fallback_name, io_as_well=False):
if not unicode_platform():
try:
unicode_name.encode(get_filesystem_encoding())
except UnicodeEncodeError:
return fallback_name
if io_as_well:
try:
unicode_name.encode(get_io_encoding())
except UnicodeEncodeError:
return fallback_name
return unicode_name
class BinTahoe(common_util.SignalMixin, unittest.TestCase, RunBinTahoeMixin):
def _check_right_code(self, file_to_check):
root_to_check = get_root_from_file(file_to_check)
if os.path.basename(root_to_check) == 'dist':
root_to_check = os.path.dirname(root_to_check)
cwd = os.path.normcase(os.path.realpath("."))
root_from_cwd = os.path.dirname(cwd)
if os.path.basename(root_from_cwd) == 'src':
root_from_cwd = os.path.dirname(root_from_cwd)
# This is needed if we are running in a temporary directory created by 'make tmpfstest'.
if os.path.basename(root_from_cwd).startswith('tmp'):
root_from_cwd = os.path.dirname(root_from_cwd)
same = (root_from_cwd == root_to_check)
if not same:
try:
same = os.path.samefile(root_from_cwd, root_to_check)
except AttributeError, e:
e # hush pyflakes
if not same:
msg = ("We seem to be testing the code at %r,\n"
"(according to the source filename %r),\n"
"but expected to be testing the code at %r.\n" %
(root_to_check, file_to_check, root_from_cwd))
root_from_cwdu = os.path.dirname(
os.path.normcase(os.path.normpath(os.getcwdu())))
if os.path.basename(root_from_cwdu) == u'src':
root_from_cwdu = os.path.dirname(root_from_cwdu)
# This is needed if we are running in a temporary directory created by 'make tmpfstest'.
if os.path.basename(root_from_cwdu).startswith(u'tmp'):
root_from_cwdu = os.path.dirname(root_from_cwdu)
if not isinstance(root_from_cwd, unicode) and root_from_cwd.decode(
get_filesystem_encoding(), 'replace') != root_from_cwdu:
msg += (
"However, this may be a false alarm because the current directory path\n"
"is not representable in the filesystem encoding. Please run the tests\n"
"from the root of the Tahoe-LAFS distribution at a non-Unicode path."
)
raise unittest.SkipTest(msg)
else:
msg += "Please run the tests from the root of the Tahoe-LAFS distribution."
self.fail(msg)
def __init__(self, client, upload_dircap, local_dir_utf8, inotify=None):
service.MultiService.__init__(self)
try:
local_dir_u = abspath_expanduser_unicode(local_dir_utf8.decode('utf-8'))
if sys.platform == "win32":
local_dir = local_dir_u
else:
local_dir = local_dir_u.encode(get_filesystem_encoding())
except (UnicodeEncodeError, UnicodeDecodeError):
raise AssertionError("The '[drop_upload] local.directory' parameter %s was not valid UTF-8 or "
"could not be represented in the filesystem encoding."
% quote_output(local_dir_utf8))
self._client = client
self._stats_provider = client.stats_provider
self._convergence = client.convergence
self._local_path = FilePath(local_dir)
self.is_upload_ready = False
if inotify is None:
from twisted.internet import inotify
self._inotify = inotify
if not self._local_path.exists():
raise AssertionError("The '[drop_upload] local.directory' parameter was %s but there is no directory at that location." % quote_output(local_dir_u))
if not self._local_path.isdir():
raise AssertionError("The '[drop_upload] local.directory' parameter was %s but the thing at that location is not a directory." % quote_output(local_dir_u))
# TODO: allow a path rather than a cap URI.
self._parent = self._client.create_node_from_uri(upload_dircap)
if not IDirectoryNode.providedBy(self._parent):
raise AssertionError("The URI in 'private/drop_upload_dircap' does not refer to a directory.")
if self._parent.is_unknown() or self._parent.is_readonly():
raise AssertionError("The URI in 'private/drop_upload_dircap' is not a writecap to a directory.")
self._uploaded_callback = lambda ign: None
self._notifier = inotify.INotify()
# We don't watch for IN_CREATE, because that would cause us to read and upload a
# possibly-incomplete file before the application has closed it. There should always
# be an IN_CLOSE_WRITE after an IN_CREATE (I think).
# TODO: what about IN_MOVE_SELF or IN_UNMOUNT?
mask = inotify.IN_CLOSE_WRITE | inotify.IN_MOVED_TO | inotify.IN_ONLYDIR
self._notifier.watch(self._local_path, mask=mask, callbacks=[self._notify])
def _test_file(self, name_u, data, temporary=False):
previously_uploaded = self._get_count('drop_upload.files_uploaded')
previously_disappeared = self._get_count('drop_upload.files_disappeared')
d = defer.Deferred()
# Note: this relies on the fact that we only get one IN_CLOSE_WRITE notification per file
# (otherwise we would get a defer.AlreadyCalledError). Should we be relying on that?
self.uploader.set_uploaded_callback(d.callback)
path_u = os.path.join(self.local_dir, name_u)
if sys.platform == "win32":
path = filepath.FilePath(path_u)
else:
path = filepath.FilePath(path_u.encode(get_filesystem_encoding()))
# We don't use FilePath.setContent() here because it creates a temporary file that
# is renamed into place, which causes events that the test is not expecting.
f = open(path.path, "wb")
try:
if temporary and sys.platform != "win32":
os.unlink(path.path)
f.write(data)
finally:
f.close()
if temporary and sys.platform == "win32":
os.unlink(path.path)
self.notify_close_write(path)
if temporary:
d.addCallback(lambda ign: self.shouldFail(NoSuchChildError, 'temp file not uploaded', None,
self.upload_dirnode.get, name_u))
d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_disappeared'),
previously_disappeared + 1))
else:
d.addCallback(lambda ign: self.upload_dirnode.get(name_u))
d.addCallback(download_to_data)
d.addCallback(lambda actual_data: self.failUnlessReallyEqual(actual_data, data))
d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_uploaded'),
previously_uploaded + 1))
d.addCallback(lambda ign: self.failUnlessReallyEqual(self._get_count('drop_upload.files_queued'), 0))
return d
def test_the_right_code(self):
# running "tahoe" in a subprocess should find the same code that
# holds this test file, else something is weird
test_path = os.path.dirname(os.path.dirname(os.path.normcase(os.path.realpath(srcfile))))
bintahoe_import_path = yield self.find_import_location()
same = (bintahoe_import_path == test_path)
if not same:
msg = ("My tests and my 'tahoe' executable are using different paths.\n"
"tahoe: %r\n"
"tests: %r\n"
"( according to the test source filename %r)\n" %
(bintahoe_import_path, test_path, srcfile))
if (not isinstance(rootdir, unicode) and
rootdir.decode(get_filesystem_encoding(), 'replace') != rootdir):
msg += ("However, this may be a false alarm because the import path\n"
"is not representable in the filesystem encoding.")
raise unittest.SkipTest(msg)
else:
msg += "Please run the tests in a virtualenv that includes both the Tahoe-LAFS library and the 'tahoe' executable."
self.fail(msg)
def start_client(self):
# this returns a Deferred that fires with the client's control.furl
log.msg("MAKING CLIENT")
# self.testdir is an absolute Unicode path
clientdir = self.clientdir = os.path.join(self.testdir, u"client")
clientdir_str = clientdir.encode(get_filesystem_encoding())
quiet = StringIO()
create_node.create_node({'basedir': clientdir}, out=quiet)
log.msg("DONE MAKING CLIENT")
# now replace tahoe.cfg
# set webport=0 and then ask the node what port it picked.
f = open(os.path.join(clientdir, "tahoe.cfg"), "w")
f.write("[node]\n"
"web.port = tcp:0:interface=127.0.0.1\n"
"[client]\n"
"introducer.furl = %s\n"
"shares.happy = 1\n"
"[storage]\n"
% (self.introducer_furl,))
if self.mode in ("upload-self", "receive"):
# accept and store shares, to trigger the memory consumption bugs
pass
else:
# don't accept any shares
f.write("readonly = true\n")
## also, if we do receive any shares, throw them away
#f.write("debug_discard = true")
if self.mode == "upload-self":
pass
f.close()
self.keepalive_file = os.path.join(clientdir,
"suicide_prevention_hotline")
# now start updating the mtime.
self.touch_keepalive()
ts = internet.TimerService(1.0, self.touch_keepalive)
ts.setServiceParent(self.sparent)
pp = ClientWatcher()
self.proc_done = pp.d = defer.Deferred()
logfile = os.path.join(self.basedir, "client.log")
cmd = ["twistd", "-n", "-y", "tahoe-client.tac", "-l", logfile]
env = os.environ.copy()
self.proc = reactor.spawnProcess(pp, cmd[0], cmd, env, path=clientdir_str)
log.msg("CLIENT STARTED")
# now we wait for the client to get started. we're looking for the
# control.furl file to appear.
furl_file = os.path.join(clientdir, "private", "control.furl")
url_file = os.path.join(clientdir, "node.url")
def _check():
if pp.ended and pp.ended.value.status != 0:
# the twistd process ends normally (with rc=0) if the child
# is successfully launched. It ends abnormally (with rc!=0)
# if the child cannot be launched.
raise ChildDidNotStartError("process ended while waiting for startup")
return os.path.exists(furl_file)
d = self.poll(_check, 0.1)
# once it exists, wait a moment before we read from it, just in case
# it hasn't finished writing the whole thing. Ideally control.furl
# would be created in some atomic fashion, or made non-readable until
# it's ready, but I can't think of an easy way to do that, and I
# think the chances that we'll observe a half-write are pretty low.
def _stall(res):
d2 = defer.Deferred()
reactor.callLater(0.1, d2.callback, None)
return d2
d.addCallback(_stall)
def _read(res):
# read the node's URL
self.webish_url = open(url_file, "r").read().strip()
if self.webish_url[-1] == "/":
# trim trailing slash, since the rest of the code wants it gone
self.webish_url = self.webish_url[:-1]
f = open(furl_file, "r")
furl = f.read()
return furl.strip()
d.addCallback(_read)
return d
def start_client(self):
# this returns a Deferred that fires with the client's control.furl
log.msg("MAKING CLIENT")
# self.testdir is an absolute Unicode path
clientdir = self.clientdir = os.path.join(self.testdir, u"client")
clientdir_str = clientdir.encode(get_filesystem_encoding())
quiet = StringIO()
create_node.create_node({'basedir': clientdir}, out=quiet)
log.msg("DONE MAKING CLIENT")
# now replace tahoe.cfg
# set webport=0 and then ask the node what port it picked.
f = open(os.path.join(clientdir, "tahoe.cfg"), "w")
f.write("[node]\n"
"web.port = tcp:0:interface=127.0.0.1\n"
"[client]\n"
"introducer.furl = %s\n"
"shares.happy = 1\n"
"[storage]\n" % (self.introducer_furl, ))
if self.mode in ("upload-self", "receive"):
# accept and store shares, to trigger the memory consumption bugs
pass
else:
# don't accept any shares
f.write("readonly = true\n")
## also, if we do receive any shares, throw them away
#f.write("debug_discard = true")
if self.mode == "upload-self":
pass
f.close()
self.keepalive_file = os.path.join(clientdir,
client.Client.EXIT_TRIGGER_FILE)
# now start updating the mtime.
self.touch_keepalive()
ts = internet.TimerService(1.0, self.touch_keepalive)
ts.setServiceParent(self.sparent)
pp = ClientWatcher()
self.proc_done = pp.d = defer.Deferred()
logfile = os.path.join(self.basedir, "client.log")
tahoes = procutils.which("tahoe")
if not tahoes:
raise RuntimeError("unable to find a 'tahoe' executable")
cmd = [tahoes[0], "run", ".", "-l", logfile]
env = os.environ.copy()
self.proc = reactor.spawnProcess(pp,
cmd[0],
cmd,
env,
path=clientdir_str)
log.msg("CLIENT STARTED")
# now we wait for the client to get started. we're looking for the
# control.furl file to appear.
furl_file = os.path.join(clientdir, "private", "control.furl")
url_file = os.path.join(clientdir, "node.url")
def _check():
if pp.ended and pp.ended.value.status != 0:
# the twistd process ends normally (with rc=0) if the child
# is successfully launched. It ends abnormally (with rc!=0)
# if the child cannot be launched.
raise ChildDidNotStartError(
"process ended while waiting for startup")
return os.path.exists(furl_file)
d = self.poll(_check, 0.1)
# once it exists, wait a moment before we read from it, just in case
# it hasn't finished writing the whole thing. Ideally control.furl
# would be created in some atomic fashion, or made non-readable until
# it's ready, but I can't think of an easy way to do that, and I
# think the chances that we'll observe a half-write are pretty low.
def _stall(res):
d2 = defer.Deferred()
reactor.callLater(0.1, d2.callback, None)
return d2
d.addCallback(_stall)
def _read(res):
# read the node's URL
self.webish_url = open(url_file, "r").read().strip()
if self.webish_url[-1] == "/":
# trim trailing slash, since the rest of the code wants it gone
self.webish_url = self.webish_url[:-1]
f = open(furl_file, "r")
furl = f.read()
return furl.strip()
d.addCallback(_read)
return d
def _publish(res):
furl_file = os.path.join(self.basedir, "private", "storage.furl").encode(get_filesystem_encoding())
furl = self.tub.registerReference(ss, furlFile=furl_file)
ri_name = RIStorageServer.__remote_name__
self.introducer_client.publish(furl, "storage", ri_name)
