def test_functional_ubuntu_1404(self):
"""
The expected deb files are generated on Ubuntu14.04.
"""
output_dir = FilePath(self.mktemp())
check_call([
FLOCKER_PATH.descendant([b'admin', b'build-package']).path,
'--destination-path', output_dir.path,
'--distribution', 'ubuntu-14.04',
FLOCKER_PATH.path
])
python_version = __version__
rpm_version = make_rpm_version(python_version)
expected_basenames = (
('clusterhq-python-flocker', 'amd64'),
('clusterhq-flocker-cli', 'all'),
('clusterhq-flocker-node', 'all'),
)
expected_filenames = []
for basename, arch in expected_basenames:
f = '{}_{}-{}_{}.deb'.format(
basename, rpm_version.version, rpm_version.release, arch)
expected_filenames.append(f)
self.assertEqual(
set(expected_filenames),
set(f.basename() for f in output_dir.children())
)
for f in output_dir.children():
assert_deb_lint(self, f)
def test_functional_centos_7(self):
"""
The expected RPM files are built for CentOS 7
"""
output_dir = FilePath(self.mktemp())
check_call([
FLOCKER_PATH.descendant([b'admin', b'build-package']).path,
'--destination-path', output_dir.path,
'--distribution', 'centos-7',
FLOCKER_PATH.path
])
python_version = __version__
rpm_version = make_rpm_version(python_version)
expected_basenames = (
('clusterhq-python-flocker', 'x86_64'),
('clusterhq-flocker-cli', 'noarch'),
('clusterhq-flocker-node', 'noarch'),
)
expected_filenames = []
for basename, arch in expected_basenames:
f = '{}-{}-{}.{}.rpm'.format(
basename, rpm_version.version, rpm_version.release, arch)
expected_filenames.append(f)
self.assertEqual(
set(expected_filenames),
set(f.basename() for f in output_dir.children())
)
for f in output_dir.children():
assert_rpm_lint(self, f)
def test_functional_ubuntu_1404(self):
"""
The expected deb files are generated on Ubuntu14.04.
"""
output_dir = FilePath(self.mktemp())
check_call([
FLOCKER_PATH.descendant(['admin', 'build-package']).path,
'--destination-path', output_dir.path, '--distribution',
'ubuntu-14.04', FLOCKER_PATH.path
])
python_version = __version__
rpm_version = make_rpm_version(python_version)
expected_basenames = (
('clusterhq-python-flocker', 'amd64'),
('clusterhq-flocker-cli', 'all'),
('clusterhq-flocker-node', 'all'),
)
expected_filenames = []
for basename, arch in expected_basenames:
f = '{}_{}-{}_{}.deb'.format(basename, rpm_version.version,
rpm_version.release, arch)
expected_filenames.append(f)
self.assertEqual(set(expected_filenames),
set(f.basename() for f in output_dir.children()))
for f in output_dir.children():
assert_deb_lint(self, f)
def main(argv):
# input
posts = FilePath(argv[1])
# output
blog = FilePath(argv[2])
# Since Sphinx gets confused by image paths with "special" characters in
# them, generate new names for all the image paths and a mapping from the
# old names to the new names.
images = FilePath(argv[3])
imagepaths = []
for post in images.children():
if post.isdir():
imagepaths.append(post)
safe = post.sibling(fixpath(post.basename()))
if post != safe and not safe.isdir():
post.moveTo(safe)
safe.linkTo(post)
entries = []
for post in posts.children():
data = post.getContent().decode("utf-8")
ignored, header, body = data.split(b"---", 2)
meta = dict((text.strip() for text in line.split(":", 1)) for line in header.splitlines() if line.strip())
date = datetime.strptime(meta["date"], "%Y/%m/%d %H:%M:%S")
parent = blog.preauthChild(
("%d/%02d/%02d" % (date.year, date.month, date.day)).encode("utf-8"))
title = fixpath(meta["title"].strip().lower().encode("utf-8")).decode("utf-8")
entry = parent.child((title + ".rst").encode("utf-8"))
header = HEADER_TEMPLATE % dict(
author=meta["author"].strip(), categories="none",
tags=meta["categories"].strip(), title=meta["title"].strip(),
underbar="=" * len(meta["title"].strip()))
for path in imagepaths:
body = body.replace(
u"/" + path.basename().decode("utf-8") + u"/",
u"/" + fixpath(path.basename()).decode("utf-8") + u"/")
if not parent.isdir():
parent.makedirs()
entry.setContent((header + html2rst(body)).encode("utf-8"))
entries.append(entry)
entries.sort()
entries.reverse()
sitemap = SITEMAP % dict(
entries="".join([
"\n " + "/".join(entry.segmentsFrom(blog))
for entry in entries]))
blog.child(b"master.rst").setContent(sitemap.encode("utf-8"))
FilePath(b"conf.py").copyTo(blog.child(b"conf.py"))
def test_functional_fedora_20(self):
"""
The expected RPM files are built for Fedora 20
"""
output_dir = FilePath(self.mktemp())
check_call([
FLOCKER_PATH.descendant(['admin', 'build-package']).path,
'--destination-path', output_dir.path, '--distribution',
'fedora-20', FLOCKER_PATH.path
])
python_version = __version__
rpm_version = make_rpm_version(python_version)
expected_basenames = (
('clusterhq-python-flocker', 'x86_64'),
('clusterhq-flocker-cli', 'noarch'),
('clusterhq-flocker-node', 'noarch'),
)
expected_filenames = []
for basename, arch in expected_basenames:
f = '{}-{}-{}.{}.rpm'.format(basename, rpm_version.version,
rpm_version.release, arch)
expected_filenames.append(f)
self.assertEqual(set(expected_filenames),
set(f.basename() for f in output_dir.children()))
for f in output_dir.children():
assert_rpm_lint(self, f)
def _large_request_test(self, request_body_size):
"""
Assert that when a request with a body of of the given size is received
its content is written to the directory the ``TahoeLAFSSite`` is
configured with.
"""
tempdir = FilePath(self.mktemp())
tempdir.makedirs()
request = self._create_request(tempdir)
# So. Bad news. The temporary file for the uploaded content is
# unnamed (and this isn't even necessarily a bad thing since it is how
# you get automatic on-process-exit cleanup behavior on POSIX). It's
# not visible by inspecting the filesystem. It has no name we can
# discover. Then how do we verify it is written to the right place?
# The question itself is meaningless if we try to be too precise. It
# *has* no filesystem location. However, it is still stored *on* some
# filesystem. We still want to make sure it is on the filesystem we
# specified because otherwise it might be on a filesystem that's too
# small or undesirable in some other way.
#
# I don't know of any way to ask a file descriptor which filesystem
# it's on, either, though. It might be the case that the [f]statvfs()
# result could be compared somehow to infer the filesystem but
# ... it's not clear what the failure modes might be there, across
# different filesystems and runtime environments.
#
# Another approach is to make the temp directory unwriteable and
# observe the failure when an attempt is made to create a file there.
# This is hardly a lovely solution but at least it's kind of simple.
#
# It would be nice if it worked consistently cross-platform but on
# Windows os.chmod is more or less broken.
if platform.isWindows():
request.gotLength(request_body_size)
self.assertThat(
tempdir.children(),
HasLength(1),
)
else:
tempdir.chmod(0o550)
with self.assertRaises(OSError) as ctx:
request.gotLength(request_body_size)
raise Exception(
"OSError not raised, instead tempdir.children() = {}".format(
tempdir.children(),
),
)
self.assertThat(
ctx.exception.errno,
Equals(EACCES),
)
def test_add_node(self):
"""
``Certificates.add_node`` generates another node certificate.
"""
output = FilePath(self.mktemp())
output.makedirs()
certificates = Certificates.generate(output, b"some-service", 2,
b"test-cluster")
certificates.add_node(3)
self.assertEqual(
{
output.child(b"cluster.crt"),
output.child(b"cluster.key"),
output.child(b"control-some-service.crt"),
output.child(b"control-some-service.key"),
output.child(b"user.crt"),
output.child(b"user.key"),
output.child(b"node-0.crt"),
output.child(b"node-0.key"),
output.child(b"node-1.crt"),
output.child(b"node-1.key"),
output.child(b"node-3.crt"),
output.child(b"node-3.key"),
},
set(output.children()),
)
def test_catchAll(self):
"""
Everything should match a catchall rule.
"""
tmpdir = FilePath(self.mktemp())
rules = [
{
'pattern': {
'foo': '*',
},
'actions': [
{'merge_yaml': '{foo}.yml'},
]
},
{
'pattern': 'all',
'actions': [
{'merge_yaml': 'extra.yml'},
]
}
]
dumper = RuleBasedFileDumper(tmpdir.path, rules)
dumper.dumpObject({
'foo': 'thefoo',
})
self.assertTrue(tmpdir.child('thefoo.yml').exists(),
"Should have matched and acted on the first rule")
dumper.dumpObject({
'bar': 'hey',
})
self.assertTrue(tmpdir.child('extra.yml').exists(),
"Should have matched and acted on the second rule")
self.assertEqual(len(tmpdir.children()), 2, "Should only have made "
"the 2 expected files")
def test_alwaysPreferPy(self):
"""
Verify that .py files will always be preferred to .pyc files, regardless of
directory listing order.
"""
mypath = FilePath(self.mktemp())
mypath.createDirectory()
pp = modules.PythonPath(sysPath=[mypath.path])
originalSmartPath = pp._smartPath
def _evilSmartPath(pathName):
o = originalSmartPath(pathName)
originalChildren = o.children
def evilChildren():
# normally this order is random; let's make sure it always
# comes up .pyc-first.
x = list(originalChildren())
x.sort()
x.reverse()
return x
o.children = evilChildren
return o
mypath.child("abcd.py").setContent(b"\n")
compileall.compile_dir(mypath.path, quiet=True)
# sanity check
self.assertEqual(len(list(mypath.children())), 2)
pp._smartPath = _evilSmartPath
self.assertEqual(pp["abcd"].filePath, mypath.child("abcd.py"))
def test_exportAllContacts(self):
"""
Run the export with --all --contacts to get a directory of addressbooks from all
addressbook homes in the database.
"""
yield populateAddressBooksFrom(
{
"user01": {
"addressbook": {
"1.vcf": adbk1Root.child("1.vcf").getContent(),
"2.vcf": adbk1Root.child("2.vcf").getContent(),
"3.vcf": adbk1Root.child("3.vcf").getContent(),
}
},
"user02": {
"addressbook": {
"1.vcf": adbk1Root.child("1.vcf").getContent(),
},
}
}, self.store)
outputDir = FilePath(self.mktemp())
outputDir.makedirs()
main([
'calendarserver_export', '--directory', outputDir.path, '--all',
'--contacts'
],
reactor=self)
yield self.waitToStop
self.assertEquals(
set(["user01_addressbook.vcf", "user02_addressbook.vcf"]),
set([child.basename() for child in outputDir.children()]))
def test_alwaysPreferPy(self):
"""
Verify that .py files will always be preferred to .pyc files, regardless of
directory listing order.
"""
mypath = FilePath(self.mktemp())
mypath.createDirectory()
pp = modules.PythonPath(sysPath=[mypath.path])
originalSmartPath = pp._smartPath
def _evilSmartPath(pathName):
o = originalSmartPath(pathName)
originalChildren = o.children
def evilChildren():
# normally this order is random; let's make sure it always
# comes up .pyc-first.
x = originalChildren()
x.sort()
x.reverse()
return x
o.children = evilChildren
return o
mypath.child("abcd.py").setContent("\n")
compileall.compile_dir(mypath.path, quiet=True)
# sanity check
self.assertEquals(len(mypath.children()), 2)
pp._smartPath = _evilSmartPath
self.assertEquals(pp["abcd"].filePath, mypath.child("abcd.py"))
def test_exportOneAddressbook(self):
"""
Run the export with a single uid and --contacts
"""
yield populateAddressBooksFrom(
{
"user01": {
"addressbook": {
"1.vcf": adbk1Root.child("1.vcf").getContent(),
"2.vcf": adbk1Root.child("2.vcf").getContent(),
"3.vcf": adbk1Root.child("3.vcf").getContent(),
}
},
"user02": {
"addressbook": {
"1.vcf": adbk1Root.child("1.vcf").getContent(),
},
}
}, self.store)
outputDir = FilePath(self.mktemp())
outputDir.makedirs()
main([
'calendarserver_export', '--directory', outputDir.path, '--uid',
'user01', '--contacts'
],
reactor=self)
yield self.waitToStop
self.assertEquals(
set(["user01_addressbook.vcf"]),
set([child.basename() for child in outputDir.children()]))
def test_exportAll(self):
"""
Run the export with --all to get a directory of calendars from all
calendar homes in the database.
"""
yield populateCalendarsFrom(
{
"user01": {
"calendar1": {
"valentines-day.ics": (valentines, {}),
"new-years-day.ics": (newYears, {})
}
},
"user02": {
"calendar1": {
"valentines-day.ics": (valentines, {})
},
"calendar2": {
"new-years-day.ics": (newYears, {})
}
}
}, self.store)
outputDir = FilePath(self.mktemp())
outputDir.makedirs()
main([
'calendarserver_export', '--directory', outputDir.path, '--all'
],
reactor=self)
yield self.waitToStop
self.assertEquals(
set([
"user01_calendar1.ics", "user02_calendar1.ics",
"user02_calendar2.ics"
]), set([child.basename() for child in outputDir.children()]))
def test_generated(self):
"""
``Certificates.generate`` generates a certificate authority
certificate, a control service certificate, a user certificate, and the
given number of node certificates.
"""
output = FilePath(self.mktemp())
output.makedirs()
Certificates.generate(output, b"some-service", 2, b"test-cluster")
self.assertEqual(
{
output.child(b"cluster.crt"),
output.child(b"cluster.key"),
output.child(b"control-some-service.crt"),
output.child(b"control-some-service.key"),
output.child(b"user.crt"),
output.child(b"user.key"),
output.child(b"node-0.crt"),
output.child(b"node-0.key"),
output.child(b"node-1.crt"),
output.child(b"node-1.key"),
},
set(output.children()),
)
class TestOpenSSHConfig(unittest.TestCase):
def setUp(self):
self.directory = FilePath(self.mktemp())
self.directory.createDirectory()
def test_files(self):
openSSHConfig.setupConfig(self.directory.path, 2222)
for file in self.directory.children():
f = file.open()
contents = f.read()
f.close()
self.assertTrue("%" not in contents)
self.assertEquals(len(self.directory.children()), 5)
def test_commandOptions(self):
for option in openSSHConfig.setupConfig(self.directory.path, 2222):
self.assertTrue("%" not in option)
def doIt(self, txn):
if raw_input("Do you really want to remove all data [y/n]: ")[0].lower() != 'y':
print "No data removed"
returnValue(None)
wipeout = (
# These are ordered in such a way as to ensure key constraints are not
# violated as data is removed
schema.RESOURCE_PROPERTY,
schema.CALENDAR_OBJECT_REVISIONS,
schema.CALENDAR,
#schema.CALENDAR_BIND, - cascades
#schema.CALENDAR_OBJECT, - cascades
#schema.TIME_RANGE, - cascades
#schema.TRANSPARENCY, - cascades
schema.CALENDAR_HOME,
#schema.CALENDAR_HOME_METADATA - cascades
schema.ATTACHMENT,
schema.ADDRESSBOOK_OBJECT_REVISIONS,
schema.ADDRESSBOOK,
#schema.ADDRESSBOOK_BIND, - cascades
#schema.ADDRESSBOOK_OBJECT, - cascades
schema.ADDRESSBOOK_HOME,
#schema.ADDRESSBOOK_HOME_METADATA, - cascades
schema.NOTIFICATION_HOME,
schema.NOTIFICATION,
#schema.NOTIFICATION_OBJECT_REVISIONS - cascades,
)
for tableschema in wipeout:
yield self.removeTableData(txn, tableschema)
print "Removed rows in table %s" % (tableschema,)
if calendaruserproxy.ProxyDBService is not None:
calendaruserproxy.ProxyDBService.clean() #@UndefinedVariable
print "Removed all proxies"
else:
print "No proxy database to clean."
fp = FilePath(config.AttachmentsRoot)
if fp.exists():
for child in fp.children():
child.remove()
print "Removed attachments."
else:
print "No attachments path to delete."
class TestOpenSSHConfig(unittest.TestCase):
def setUp(self):
self.directory = FilePath(self.mktemp())
self.directory.createDirectory()
def test_files(self):
openSSHConfig.setupConfig(self.directory.path, 2222)
for file in self.directory.children():
f = file.open()
contents = f.read()
f.close()
self.assertTrue("%" not in contents)
self.assertEquals(len(self.directory.children()), 5)
def test_commandOptions(self):
for option in openSSHConfig.setupConfig(self.directory.path, 2222):
self.assertTrue("%" not in option)
def test_unpartialize(self):
xpg = self.xpg_with_backup_dir()
folder = FilePath(xpg.archive_log_directory)
folder.child("1").touch()
folder.child("2").touch()
folder.child("3").touch()
folder.child("4.partial").touch()
xpg.unpartialize()
self.assertEquals(set([c.basename() for c in folder.children()]),
set(['1', '2', '3', '4']))
def test_unpartialize(self):
xpg = self.xpg_with_backup_dir()
folder = FilePath(xpg.archive_log_directory)
folder.child("1").touch()
folder.child("2").touch()
folder.child("3").touch()
folder.child("4.partial").touch()
xpg.unpartialize()
self.assertEquals(set([c.basename() for c in folder.children()]),
set(['1', '2', '3', '4']))
def doIt(self, txn):
if raw_input("Do you really want to remove all data [y/n]: ")[0].lower() != 'y':
print("No data removed")
returnValue(None)
wipeout = (
# These are ordered in such a way as to ensure key constraints are not
# violated as data is removed
schema.RESOURCE_PROPERTY,
schema.CALENDAR_OBJECT_REVISIONS,
schema.CALENDAR,
#schema.CALENDAR_BIND, - cascades
#schema.CALENDAR_OBJECT, - cascades
#schema.TIME_RANGE, - cascades
#schema.TRANSPARENCY, - cascades
schema.CALENDAR_HOME,
#schema.CALENDAR_HOME_METADATA - cascades
schema.ATTACHMENT,
schema.ADDRESSBOOK_OBJECT_REVISIONS,
schema.ADDRESSBOOK_HOME,
#schema.ADDRESSBOOK_HOME_METADATA, - cascades
#schema.ADDRESSBOOK_BIND, - cascades
#schema.ADDRESSBOOK_OBJECT, - cascades
schema.NOTIFICATION_HOME,
schema.NOTIFICATION,
#schema.NOTIFICATION_OBJECT_REVISIONS - cascades,
)
for tableschema in wipeout:
yield self.removeTableData(txn, tableschema)
print("Removed rows in table %s" % (tableschema,))
if calendaruserproxy.ProxyDBService is not None:
calendaruserproxy.ProxyDBService.clean() #@UndefinedVariable
print("Removed all proxies")
else:
print("No proxy database to clean.")
fp = FilePath(config.AttachmentsRoot)
if fp.exists():
for child in fp.children():
child.remove()
print("Removed attachments.")
else:
print("No attachments path to delete.")
def loadServices(base):
from braid import config
services = {}
servicesDir = FilePath(base).sibling('services')
for serviceDir in servicesDir.children():
serviceName = serviceDir.basename()
fabfile = serviceDir.child('fabfile.py')
if fabfile.exists():
module = imp.load_source(serviceName, fabfile.path, fabfile.open())
if module.config == config:
del module.config
services[serviceName] = module
return services
def loadServices(base):
from braid import config
services = {}
servicesDir = FilePath(base).sibling('services')
for serviceDir in servicesDir.children():
serviceName = serviceDir.basename()
fabfile = serviceDir.child('fabfile.py')
if fabfile.exists():
module = imp.load_source(serviceName, fabfile.path, fabfile.open())
if module.config == config:
del module.config
services[serviceName] = module
return services
def assertExtractedStructure(self, outputFile, dirDict):
"""
Assert that a tarfile content is equivalent to one described by a dict.
@param outputFile: The tar file built by L{DistributionBuilder}.
@type outputFile: L{FilePath}.
@param dirDict: The dict that should describe the contents of the
directory. It should be the same structure as the C{dirDict}
parameter to L{createStructure}.
@type dirDict: C{dict}
"""
tarFile = tarfile.TarFile.open(outputFile.path, "r:bz2")
extracted = FilePath(self.mktemp())
extracted.createDirectory()
for info in tarFile:
tarFile.extract(info, path=extracted.path)
self.assertStructure(extracted.children()[0], dirDict)
def test_log_pruning(self):
"""
L{XPostgres.prune_useless_archive_logs} will remove unhelpful log
files.
"""
xpg = self.xpg_with_backup_dir()
folder = FilePath(xpg.archive_log_directory)
folder.child("base_backup").createDirectory()
folder.child("unknown").touch()
folder.child("foo.bar.partial").touch()
folder.child("foo.bar").touch()
(folder.child("something").temporarySibling(
".in-progress").open().close())
xpg.prune_useless_archive_logs()
self.assertEquals(set([c.basename() for c in folder.children()]),
set(['base_backup', 'unknown', 'foo.bar']))
def test_log_pruning(self):
"""
L{XPostgres.prune_useless_archive_logs} will remove unhelpful log
files.
"""
xpg = self.xpg_with_backup_dir()
folder = FilePath(xpg.archive_log_directory)
folder.child("base_backup").createDirectory()
folder.child("unknown").touch()
folder.child("foo.bar.partial").touch()
folder.child("foo.bar").touch()
(folder.child("something").temporarySibling(".in-progress")
.open().close())
xpg.prune_useless_archive_logs()
self.assertEquals(set([c.basename() for c in folder.children()]),
set(['base_backup', 'unknown', 'foo.bar']))
def assertExtractedStructure(self, outputFile, dirDict):
"""
Assert that a tarfile content is equivalent to one described by a dict.
@param outputFile: The tar file built by L{DistributionBuilder}.
@type outputFile: L{FilePath}.
@param dirDict: The dict that should describe the contents of the
directory. It should be the same structure as the C{dirDict}
parameter to L{createStructure}.
@type dirDict: C{dict}
"""
tarFile = tarfile.TarFile.open(outputFile.path, "r:bz2")
extracted = FilePath(self.mktemp())
extracted.createDirectory()
for info in tarFile:
tarFile.extract(info, path=extracted.path)
self.assertStructure(extracted.children()[0], dirDict)
def test_generated(self):
"""
``Certificates.generate`` generates a certificate authority
certificate, a control service certificate, a user certificate, and the
given number of node certificates.
"""
output = FilePath(self.mktemp())
output.makedirs()
Certificates.generate(output, b"some-service", 2, b"test-cluster")
self.assertEqual(
{
output.child(b"cluster.crt"), output.child(b"cluster.key"),
output.child(b"control-some-service.crt"),
output.child(b"control-some-service.key"),
output.child(b"user.crt"), output.child(b"user.key"),
output.child(b"node-0.crt"), output.child(b"node-0.key"),
output.child(b"node-1.crt"), output.child(b"node-1.key"),
},
set(output.children()),
)
def test_add_node(self):
"""
``Certificates.add_node`` generates another node certificate.
"""
output = FilePath(self.mktemp())
output.makedirs()
certificates = Certificates.generate(output, b"some-service", 2,
b"test-cluster")
certificates.add_node(3)
self.assertEqual(
{
output.child(b"cluster.crt"), output.child(b"cluster.key"),
output.child(b"control-some-service.crt"),
output.child(b"control-some-service.key"),
output.child(b"user.crt"), output.child(b"user.key"),
output.child(b"node-0.crt"), output.child(b"node-0.key"),
output.child(b"node-1.crt"), output.child(b"node-1.key"),
output.child(b"node-3.crt"), output.child(b"node-3.key"),
},
set(output.children()),
)
class NewsBuilderTests(TestCase, StructureAssertingMixin):
"""
Tests for L{NewsBuilder}.
"""
skip = svnSkip
def setUp(self):
"""
Create a fake project and stuff some basic structure and content into
it.
"""
self.builder = NewsBuilder()
self.project = FilePath(self.mktemp())
self.project.createDirectory()
self.existingText = 'Here is stuff which was present previously.\n'
self.createStructure(
self.project, {
'NEWS':
self.existingText,
'5.feature':
'We now support the web.\n',
'12.feature':
'The widget is more robust.\n',
'15.feature':
('A very long feature which takes many words to '
'describe with any accuracy was introduced so that '
'the line wrapping behavior of the news generating '
'code could be verified.\n'),
'16.feature':
('A simpler feature\ndescribed on multiple lines\n'
'was added.\n'),
'23.bugfix':
'Broken stuff was fixed.\n',
'25.removal':
'Stupid stuff was deprecated.\n',
'30.misc':
'',
'35.misc':
'',
'40.doc':
'foo.bar.Baz.quux',
'41.doc':
'writing Foo servers'
})
def svnCommit(self, project=None):
"""
Make the C{project} directory a valid subversion directory with all
files committed.
"""
if project is None:
project = self.project
repositoryPath = self.mktemp()
repository = FilePath(repositoryPath)
runCommand(["svnadmin", "create", repository.path])
runCommand(
["svn", "checkout", "file://" + repository.path, project.path])
runCommand(["svn", "add"] + glob.glob(project.path + "/*"))
runCommand(["svn", "commit", project.path, "-m", "yay"])
def test_today(self):
"""
L{NewsBuilder._today} returns today's date in YYYY-MM-DD form.
"""
self.assertEqual(self.builder._today(),
date.today().strftime('%Y-%m-%d'))
def test_findFeatures(self):
"""
When called with L{NewsBuilder._FEATURE}, L{NewsBuilder._findChanges}
returns a list of bugfix ticket numbers and descriptions as a list of
two-tuples.
"""
features = self.builder._findChanges(self.project,
self.builder._FEATURE)
self.assertEqual(features, [
(5, "We now support the web."), (12, "The widget is more robust."),
(15, "A very long feature which takes many words to describe with "
"any accuracy was introduced so that the line wrapping behavior "
"of the news generating code could be verified."),
(16, "A simpler feature described on multiple lines was added.")
])
def test_findBugfixes(self):
"""
When called with L{NewsBuilder._BUGFIX}, L{NewsBuilder._findChanges}
returns a list of bugfix ticket numbers and descriptions as a list of
two-tuples.
"""
bugfixes = self.builder._findChanges(self.project,
self.builder._BUGFIX)
self.assertEqual(bugfixes, [(23, 'Broken stuff was fixed.')])
def test_findRemovals(self):
"""
When called with L{NewsBuilder._REMOVAL}, L{NewsBuilder._findChanges}
returns a list of removal/deprecation ticket numbers and descriptions
as a list of two-tuples.
"""
removals = self.builder._findChanges(self.project,
self.builder._REMOVAL)
self.assertEqual(removals, [(25, 'Stupid stuff was deprecated.')])
def test_findDocumentation(self):
"""
When called with L{NewsBuilder._DOC}, L{NewsBuilder._findChanges}
returns a list of documentation ticket numbers and descriptions as a
list of two-tuples.
"""
doc = self.builder._findChanges(self.project, self.builder._DOC)
self.assertEqual(doc, [(40, 'foo.bar.Baz.quux'),
(41, 'writing Foo servers')])
def test_findMiscellaneous(self):
"""
When called with L{NewsBuilder._MISC}, L{NewsBuilder._findChanges}
returns a list of removal/deprecation ticket numbers and descriptions
as a list of two-tuples.
"""
misc = self.builder._findChanges(self.project, self.builder._MISC)
self.assertEqual(misc, [(30, ''), (35, '')])
def test_writeHeader(self):
"""
L{NewsBuilder._writeHeader} accepts a file-like object opened for
writing and a header string and writes out a news file header to it.
"""
output = StringIO()
self.builder._writeHeader(output, "Super Awesometastic 32.16")
self.assertEqual(
output.getvalue(), "Super Awesometastic 32.16\n"
"=========================\n"
"\n")
def test_writeSection(self):
"""
L{NewsBuilder._writeSection} accepts a file-like object opened for
writing, a section name, and a list of ticket information (as returned
by L{NewsBuilder._findChanges}) and writes out a section header and all
of the given ticket information.
"""
output = StringIO()
self.builder._writeSection(
output, "Features",
[(3, "Great stuff."),
(17, "Very long line which goes on and on and on, seemingly "
"without end until suddenly without warning it does end.")])
self.assertEqual(
output.getvalue(), "Features\n"
"--------\n"
" - Great stuff. (#3)\n"
" - Very long line which goes on and on and on, seemingly "
"without end\n"
" until suddenly without warning it does end. (#17)\n"
"\n")
def test_writeMisc(self):
"""
L{NewsBuilder._writeMisc} accepts a file-like object opened for
writing, a section name, and a list of ticket information (as returned
by L{NewsBuilder._findChanges} and writes out a section header and all
of the ticket numbers, but excludes any descriptions.
"""
output = StringIO()
self.builder._writeMisc(output, "Other",
[(x, "") for x in range(2, 50, 3)])
self.assertEqual(
output.getvalue(), "Other\n"
"-----\n"
" - #2, #5, #8, #11, #14, #17, #20, #23, #26, #29, #32, #35, "
"#38, #41,\n"
" #44, #47\n"
"\n")
def test_build(self):
"""
L{NewsBuilder.build} updates a NEWS file with new features based on the
I{<ticket>.feature} files found in the directory specified.
"""
self.builder.build(self.project, self.project.child('NEWS'),
"Super Awesometastic 32.16")
results = self.project.child('NEWS').getContent()
self.assertEqual(
results, 'Super Awesometastic 32.16\n'
'=========================\n'
'\n'
'Features\n'
'--------\n'
' - We now support the web. (#5)\n'
' - The widget is more robust. (#12)\n'
' - A very long feature which takes many words to describe '
'with any\n'
' accuracy was introduced so that the line wrapping behavior '
'of the\n'
' news generating code could be verified. (#15)\n'
' - A simpler feature described on multiple lines was '
'added. (#16)\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Improved Documentation\n'
'----------------------\n'
' - foo.bar.Baz.quux (#40)\n'
' - writing Foo servers (#41)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n'
'Other\n'
'-----\n'
' - #30, #35\n'
'\n\n' + self.existingText)
def test_emptyProjectCalledOut(self):
"""
If no changes exist for a project, I{NEWS} gains a new section for
that project that includes some helpful text about how there were no
interesting changes.
"""
project = FilePath(self.mktemp()).child("twisted")
project.makedirs()
self.createStructure(project, {'NEWS': self.existingText})
self.builder.build(project, project.child('NEWS'),
"Super Awesometastic 32.16")
results = project.child('NEWS').getContent()
self.assertEqual(
results, 'Super Awesometastic 32.16\n'
'=========================\n'
'\n' + self.builder._NO_CHANGES + '\n\n' + self.existingText)
def test_preserveTicketHint(self):
"""
If a I{NEWS} file begins with the two magic lines which point readers
at the issue tracker, those lines are kept at the top of the new file.
"""
news = self.project.child('NEWS')
news.setContent(
'Ticket numbers in this file can be looked up by visiting\n'
'http://twistedmatrix.com/trac/ticket/<number>\n'
'\n'
'Blah blah other stuff.\n')
self.builder.build(self.project, news, "Super Awesometastic 32.16")
self.assertEqual(
news.getContent(),
'Ticket numbers in this file can be looked up by visiting\n'
'http://twistedmatrix.com/trac/ticket/<number>\n'
'\n'
'Super Awesometastic 32.16\n'
'=========================\n'
'\n'
'Features\n'
'--------\n'
' - We now support the web. (#5)\n'
' - The widget is more robust. (#12)\n'
' - A very long feature which takes many words to describe '
'with any\n'
' accuracy was introduced so that the line wrapping behavior '
'of the\n'
' news generating code could be verified. (#15)\n'
' - A simpler feature described on multiple lines was '
'added. (#16)\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Improved Documentation\n'
'----------------------\n'
' - foo.bar.Baz.quux (#40)\n'
' - writing Foo servers (#41)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n'
'Other\n'
'-----\n'
' - #30, #35\n'
'\n\n'
'Blah blah other stuff.\n')
def test_emptySectionsOmitted(self):
"""
If there are no changes of a particular type (feature, bugfix, etc), no
section for that type is written by L{NewsBuilder.build}.
"""
for ticket in self.project.children():
if ticket.splitext()[1] in ('.feature', '.misc', '.doc'):
ticket.remove()
self.builder.build(self.project, self.project.child('NEWS'),
'Some Thing 1.2')
self.assertEqual(
self.project.child('NEWS').getContent(), 'Some Thing 1.2\n'
'==============\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n\n'
'Here is stuff which was present previously.\n')
def test_duplicatesMerged(self):
"""
If two change files have the same contents, they are merged in the
generated news entry.
"""
def feature(s):
return self.project.child(s + '.feature')
feature('5').copyTo(feature('15'))
feature('5').copyTo(feature('16'))
self.builder.build(self.project, self.project.child('NEWS'),
'Project Name 5.0')
self.assertEqual(
self.project.child('NEWS').getContent(), 'Project Name 5.0\n'
'================\n'
'\n'
'Features\n'
'--------\n'
' - We now support the web. (#5, #15, #16)\n'
' - The widget is more robust. (#12)\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Improved Documentation\n'
'----------------------\n'
' - foo.bar.Baz.quux (#40)\n'
' - writing Foo servers (#41)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n'
'Other\n'
'-----\n'
' - #30, #35\n'
'\n\n'
'Here is stuff which was present previously.\n')
def createFakeTwistedProject(self):
"""
Create a fake-looking Twisted project to build from.
"""
project = FilePath(self.mktemp()).child("twisted")
project.makedirs()
self.createStructure(
project, {
'NEWS': 'Old boring stuff from the past.\n',
'_version.py': genVersion("twisted", 1, 2, 3),
'topfiles': {
'NEWS': 'Old core news.\n',
'3.feature': 'Third feature addition.\n',
'5.misc': ''
},
'conch': {
'_version.py': genVersion("twisted.conch", 3, 4, 5),
'topfiles': {
'NEWS': 'Old conch news.\n',
'7.bugfix': 'Fixed that bug.\n'
}
}
})
return project
def test_buildAll(self):
"""
L{NewsBuilder.buildAll} calls L{NewsBuilder.build} once for each
subproject, passing that subproject's I{topfiles} directory as C{path},
the I{NEWS} file in that directory as C{output}, and the subproject's
name as C{header}, and then again for each subproject with the
top-level I{NEWS} file for C{output}. Blacklisted subprojects are
skipped.
"""
builds = []
builder = NewsBuilder()
builder.build = lambda path, output, header: builds.append(
(path, output, header))
builder._today = lambda: '2009-12-01'
project = self.createFakeTwistedProject()
self.svnCommit(project)
builder.buildAll(project)
coreTopfiles = project.child("topfiles")
coreNews = coreTopfiles.child("NEWS")
coreHeader = "Twisted Core 1.2.3 (2009-12-01)"
conchTopfiles = project.child("conch").child("topfiles")
conchNews = conchTopfiles.child("NEWS")
conchHeader = "Twisted Conch 3.4.5 (2009-12-01)"
aggregateNews = project.child("NEWS")
self.assertEqual(builds, [(conchTopfiles, conchNews, conchHeader),
(conchTopfiles, aggregateNews, conchHeader),
(coreTopfiles, coreNews, coreHeader),
(coreTopfiles, aggregateNews, coreHeader)])
def test_buildAllAggregate(self):
"""
L{NewsBuilder.buildAll} aggregates I{NEWS} information into the top
files, only deleting fragments once it's done.
"""
builder = NewsBuilder()
project = self.createFakeTwistedProject()
self.svnCommit(project)
builder.buildAll(project)
aggregateNews = project.child("NEWS")
aggregateContent = aggregateNews.getContent()
self.assertIn("Third feature addition", aggregateContent)
self.assertIn("Fixed that bug", aggregateContent)
self.assertIn("Old boring stuff from the past", aggregateContent)
def test_changeVersionInNews(self):
"""
L{NewsBuilder._changeVersions} gets the release date for a given
version of a project as a string.
"""
builder = NewsBuilder()
builder._today = lambda: '2009-12-01'
project = self.createFakeTwistedProject()
self.svnCommit(project)
builder.buildAll(project)
newVersion = Version('TEMPLATE', 7, 7, 14)
coreNews = project.child('topfiles').child('NEWS')
# twisted 1.2.3 is the old version.
builder._changeNewsVersion(coreNews, "Core",
Version("twisted", 1, 2,
3), newVersion, '2010-01-01')
expectedCore = ('Twisted Core 7.7.14 (2010-01-01)\n'
'================================\n'
'\n'
'Features\n'
'--------\n'
' - Third feature addition. (#3)\n'
'\n'
'Other\n'
'-----\n'
' - #5\n\n\n')
self.assertEqual(expectedCore + 'Old core news.\n',
coreNews.getContent())
def test_removeNEWSfragments(self):
"""
L{NewsBuilder.buildALL} removes all the NEWS fragments after the build
process, using the C{svn} C{rm} command.
"""
builder = NewsBuilder()
project = self.createFakeTwistedProject()
self.svnCommit(project)
builder.buildAll(project)
self.assertEqual(5, len(project.children()))
output = runCommand(["svn", "status", project.path])
removed = [
line for line in output.splitlines() if line.startswith("D ")
]
self.assertEqual(3, len(removed))
def test_checkSVN(self):
"""
L{NewsBuilder.buildAll} raises L{NotWorkingDirectory} when the given
path is not a SVN checkout.
"""
self.assertRaises(NotWorkingDirectory, self.builder.buildAll,
self.project)
def process_fds():
path = FilePath(b"/proc/self/fd")
return set([child.basename() for child in path.children()])
class Voluminous(object):
lockFactory = DockerLock
def __init__(self, directory):
self._directory = FilePath(directory)
self._output = []
self.lock = self.lockFactory()
self.commitDatabase = JsonCommitDatabase(self._directory)
def output(self, s):
self._output.append(s)
print s
def getOutput(self):
return self._output
def allBranches(self, volume):
volumePath = self._directory.child(volume)
branches = volumePath.child("branches").children()
return [b.basename() for b in branches if b.isdir()]
def listBranches(self):
volume = self.volume()
branches = self.allBranches(volume)
currentBranch = self.getActiveBranch(volume)
self.output("\n".join(sorted(("*" if b == currentBranch else " ") + " " + b for b in branches)))
def checkoutBranch(self, branch, create):
"""
"Check out" a branch, restarting containers in process, creating it
from current branch HEAD if requested.
"""
volume = self.volume()
volumePath = self._directory.child(volume)
# this raises an exception if branch is not a valid path segment
branchPath = volumePath.child("branches").child(branch)
if create:
if branchPath.exists():
self.output("Cannot create existing branch %s" % (branch,))
return
else:
try:
HEAD = self._resolveNamedCommitCurrentBranch("HEAD", volume)
except IndexError:
self.output("You must commit ('dvol commit') before you can " "branch ('dvol checkout -b')")
return
# Copy metadata
meta = self.commitDatabase.read(volume, self.getActiveBranch(volume))
self.commitDatabase.write(volume, branch, meta)
# Then copy latest HEAD of branch into new branch data
# directory
copyTo(volumePath.child("commits").child(HEAD), branchPath)
else:
if not branchPath.exists():
self.output("Cannot switch to non-existing branch %s" % (branch,))
return
# Got here, so switch to the (maybe new branch)
self.setActiveBranch(volume, branch)
def createBranch(self, volume, branch):
branchDir = self._directory.child(volume).child("branches").child(branch)
branchDir.makedirs()
# This branchDir is the one which will be bind-mounted into running
# containers, via a symlink, but with symlinks and docker bind-mounts
# it seems that it's the permissions of the target which affects the
# (e.g.) writeability of the resulting mount.
# Because some containers have processes which run as non-root users,
# make the volume world-writeable so that it can still be useful to
# those processes. In the future, it might be better to have options
# for which uid, gid and perms the volume should have. This is
# effectively the same as `chmod a=rwx branchDir.path`.
os.chmod(branchDir.path, 0777)
self.output("Created branch %s/%s" % (volume, branch))
def createVolume(self, name):
if self._directory.child(name).exists():
self.output("Error: volume %s already exists" % (name,))
raise VolumeAlreadyExists()
self._directory.child(name).makedirs()
self.setActiveVolume(name)
self.output("Created volume %s" % (name,))
self.createBranch(name, DEFAULT_BRANCH)
def removeVolume(self, volume):
if not self._directory.child(volume).exists():
raise UsageError("Volume %r does not exist, cannot remove it" % (volume,))
containers = self.lock.containers.get_related_containers(volume)
if containers:
raise UsageError(
"Cannot remove %r while it is in use by '%s'" % (volume, (",".join(c["Name"] for c in containers)))
)
if self._userIsSure("This will remove all containers using the volume"):
self.output("Deleting volume %r" % (volume,))
# Remove related containers
self.lock.containers.remove_related_containers(volume)
self._directory.child(volume).remove()
else:
self.output("Aborting.")
def deleteBranch(self, branch):
volume = self.volume()
if branch == self.getActiveBranch(volume):
raise UsageError("Cannot delete active branch, use " "'dvol checkout' to switch branches first")
if branch not in self.allBranches(volume):
raise UsageError("Branch %r does not exist" % (branch,))
if self._userIsSure():
self.output("Deleting branch %r" % (branch,))
volumePath = self._directory.child(volume)
branchPath = volumePath.child("branches").child(branch)
branchPath.remove()
else:
self.output("Aborting.")
def _userIsSure(self, extraMessage=None):
message = "Are you sure? "
if extraMessage:
message += extraMessage
message += " (y/n): "
sys.stdout.write(message)
sys.stdout.flush()
return raw_input().lower() in ("y", "yes")
def setActiveVolume(self, volume):
self._directory.child("current_volume.json").setContent(json.dumps(dict(current_volume=volume)))
def volume(self):
currentVolume = self._directory.child("current_volume.json")
if currentVolume.exists():
volume = json.loads(currentVolume.getContent())["current_volume"]
else:
raise UsageError("No active volume: use dvol switch to choose one")
if not self._directory.child(volume).exists():
raise UsageError("Active volume %s does not exist: " "use dvol switch to choose another" % (volume,))
return volume
def setActiveBranch(self, volume, branch):
self._directory.child(volume).child("current_branch.json").setContent(json.dumps(dict(current_branch=branch)))
self.lock.acquire(volume)
try:
self.updateRunningPoint(volume)
finally:
self.lock.release(volume)
def getActiveBranch(self, volume):
currentBranch = self._directory.child(self.volume()).child("current_branch.json")
if currentBranch.exists():
return json.loads(currentBranch.getContent())["current_branch"]
else:
return DEFAULT_BRANCH
def updateRunningPoint(self, volume):
"""
construct a stable (wrt switching branches) path with symlinks
"""
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
stablePath = volumePath.child("running_point")
if stablePath.exists():
stablePath.remove()
branchPath.linkTo(stablePath)
return stablePath.path
def commitVolume(self, message):
volume = self.volume()
commitId = (str(uuid.uuid4()) + str(uuid.uuid4())).replace("-", "")[:40]
self.output(commitId)
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
commitPath = volumePath.child("commits").child(commitId)
if commitPath.exists():
raise Exception("woah, random uuid collision. try again!")
# Make the commits directory if necessary
if not commitPath.parent().exists():
commitPath.parent().makedirs()
# acquire lock (read: stop containers) to ensure consistent snapshot
# with file-copy based backend
# XXX tests for acquire/release
self.lock.acquire(volume)
try:
copyTo(branchPath, commitPath)
finally:
self.lock.release(volume)
self._recordCommit(volume, branchName, commitId, message)
def _recordCommit(self, volume, branch, commitId, message):
commitData = self.commitDatabase.read(volume, branch)
commitData.append(dict(id=commitId, message=message))
self.commitDatabase.write(volume, branch, commitData)
def exists(self, volume):
volumePath = self._directory.child(volume)
return volumePath.exists()
def listVolumes(self):
table = get_table()
table.set_cols_align(["l", "l", "l"])
dc = self.lock.containers # XXX ugly
volumes = [v for v in self._directory.children() if v.isdir()]
activeVolume = None
if volumes:
try:
activeVolume = self.volume()
except UsageError:
# don't refuse to list volumes just because none of them are active
pass
rows = (
[["", "", ""]]
+ [[" VOLUME", "BRANCH", "CONTAINERS"]]
+ [
[
("*" if v.basename() == activeVolume else " ") + " " + v.basename(),
self.getActiveBranch(v.basename()),
",".join(c["Name"] for c in dc.get_related_containers(v.basename())),
]
for v in volumes
]
)
table.add_rows(rows)
self.output(table.draw())
def listCommits(self, branch=None):
if branch is None:
branch = self.getActiveBranch(self.volume())
volume = self.volume()
aggregate = []
for commit in reversed(self.commitDatabase.read(volume, branch)):
# TODO fill in author/date
aggregate.append(
"commit %(id)s\n"
"Author: Who knows <mystery@person>\n"
"Date: Whenever\n"
"\n"
" %(message)s\n" % commit
)
self.output("\n".join(aggregate))
def _resolveNamedCommitCurrentBranch(self, commit, volume):
branch = self.getActiveBranch(volume)
remainder = commit[len("HEAD") :]
if remainder == "^" * len(remainder):
offset = len(remainder)
else:
raise UsageError("Malformed commit identifier %r" % (commit,))
commits = self.commitDatabase.read(volume, branch)
# commits are appended to, so the last one is the latest
return commits[-1 - offset]["id"]
def _destroyNewerCommits(self, commit, volume):
# TODO in the future, we'll care more about the following being an
# atomic operation
branch = self.getActiveBranch(volume)
commits = self.commitDatabase.read(volume, branch)
commitIndex = [c["id"] for c in commits].index(commit) + 1
remainingCommits = commits[:commitIndex]
destroyCommits = commits[commitIndex:]
# look in all branches for commit references before removing them
totalCommits = set()
for otherBranch in self.allBranches(volume):
if otherBranch == branch:
# skip this branch, otherwise we'll never destroy any commits
continue
commits = self.commitDatabase.read(volume, branch)
totalCommits.update(commit["id"] for commit in commits)
for commit in destroyCommits:
commitId = commit["id"]
if commitId in totalCommits:
# skip destroying this commit; it is still actively referred to
# in another branch
continue
volumePath = self._directory.child(volume)
commitPath = volumePath.child("commits").child(commitId)
commitPath.remove()
self.commitDatabase.write(volume, branch, remainingCommits)
def resetVolume(self, commit):
"""
Forcefully roll back the current working copy to this commit,
destroying any later commits.
"""
volume = self.volume()
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
if commit.startswith("HEAD"):
try:
commit = self._resolveNamedCommitCurrentBranch(commit, volume)
except IndexError:
self.output("Referenced commit does not exist; check dvol log")
return
commitPath = volumePath.child("commits").child(commit)
if not commitPath.exists():
raise NoSuchCommit("commit '%s' does not exist" % (commit,))
self.lock.acquire(volume)
try:
branchPath.remove()
copyTo(commitPath, branchPath)
self._destroyNewerCommits(commit, volume)
finally:
self.lock.release(volume)
def seedVolumes(self, compose_file):
# XXX: does not work with absolute paths, but should
compose = yaml.load(PWD_PATH.child(compose_file).open())
valid_volumes = []
# TODO: will need updating for new compose file format
for service, config in compose.iteritems():
if "volume_driver" in config and config["volume_driver"] == "dvol":
for volume in config["volumes"]:
if ":" in volume and not volume.startswith("/"):
valid_volumes.append((volume, config))
if not valid_volumes:
print 'No volumes found with "volume_driver: dvol" and a named volume (like "volumename:/path_inside_container")! Please check your docker-compose.yml file.'
else:
print "Please seed your dvol volume(s) by running the following command(s):"
for volume, config in valid_volumes:
# TODO: need some validation before running commands with string interpolation here, docker-compose file could be malicious
# TODO: would be better if we ran the command for the user, rather than making them copy and paste
print "docker run --volume-driver=dvol -v %(volume)s:/_target -ti %(image)s sh -c 'cp -av %(source)s/* /_target/'" % dict(
volume=volume.split(":")[0], source=volume.split(":")[1], image=config["image"]
)
class Voluminous(object):
def __init__(self, directory, lockFactory=DockerLock):
self._directory = FilePath(directory)
self._output = []
self.lock = lockFactory()
self.commitDatabase = JsonCommitDatabase(self._directory)
def output(self, s):
self._output.append(s)
print s
def getOutput(self):
result = ["\n".join(self._output)]
self._output = []
return result
def allBranches(self, volume):
volumePath = self._directory.child(volume)
branches = volumePath.child("branches").children()
return [b.basename() for b in branches if b.isdir()]
def listBranches(self):
volume = self.volume()
branches = self.allBranches(volume)
currentBranch = self.getActiveBranch(volume)
self.output("\n".join(
sorted(("*" if b == currentBranch else " ") + " " + b
for b in branches)))
def checkoutBranch(self, branch, create):
"""
"Check out" a branch, restarting containers in process, creating it
from current branch HEAD if requested.
"""
volume = self.volume()
volumePath = self._directory.child(volume)
# this raises an exception if branch is not a valid path segment
branchPath = volumePath.child("branches").child(branch)
if create:
if branchPath.exists():
self.output("Cannot create existing branch %s" % (branch, ))
return
else:
try:
HEAD = self._resolveNamedCommitCurrentBranch(
"HEAD", volume)
except IndexError:
self.output(
"You must commit ('dvol commit') before you can "
"branch ('dvol checkout -b')")
return
# Copy metadata
meta = self.commitDatabase.read(volume,
self.getActiveBranch(volume))
self.commitDatabase.write(volume, branch, meta)
# Then copy latest HEAD of branch into new branch data
# directory
copyTo(volumePath.child("commits").child(HEAD), branchPath)
else:
if not branchPath.exists():
self.output("Cannot switch to non-existing branch %s" %
(branch, ))
return
# Got here, so switch to the (maybe new branch)
self.setActiveBranch(volume, branch)
def createBranch(self, volume, branch):
branchDir = self._directory.child(volume).child("branches").child(
branch)
branchDir.makedirs()
# This branchDir is the one which will be bind-mounted into running
# containers, via a symlink, but with symlinks and docker bind-mounts
# it seems that it's the permissions of the target which affects the
# (e.g.) writeability of the resulting mount.
# Because some containers have processes which run as non-root users,
# make the volume world-writeable so that it can still be useful to
# those processes. In the future, it might be better to have options
# for which uid, gid and perms the volume should have. This is
# effectively the same as `chmod a=rwx branchDir.path`.
os.chmod(branchDir.path, 0777)
self.output("Created branch %s/%s" % (volume, branch))
def createVolume(self, name):
try:
# XXX: Behaviour around names with relative path identifiers
# such as '..' and '.' is largely undefined, these should
# probably be rejected outright.
if self._directory.child(name).exists():
self.output("Error: volume %s already exists" % (name, ))
return
except InsecurePath:
self.output("Error: %s is not a valid name" % (name, ))
return
self._directory.child(name).makedirs()
self.setActiveVolume(name)
self.output("Created volume %s" % (name, ))
self.createBranch(name, DEFAULT_BRANCH)
def removeVolume(self, volume, force=False):
try:
if not self._directory.child(volume).exists():
self.output("Volume %r does not exist, cannot remove it" %
(volume, ))
return
except InsecurePath:
self.output("Error: %s is not a valid name" % (volume, ))
return
containers = self.lock.containers.get_related_containers(volume)
if containers:
raise UsageError("Cannot remove %r while it is in use by '%s'" %
(volume, (",".join(c['Name']
for c in containers))))
if force or self._userIsSure(
"This will remove all containers using the volume"):
self.output("Deleting volume %r" % (volume, ))
# Remove related containers
self.lock.containers.remove_related_containers(volume)
self._directory.child(volume).remove()
else:
self.output("Aborting.")
def deleteBranch(self, branch):
volume = self.volume()
if branch == self.getActiveBranch(volume):
raise UsageError("Cannot delete active branch, use "
"'dvol checkout' to switch branches first")
if branch not in self.allBranches(volume):
raise UsageError("Branch %r does not exist" % (branch, ))
if self._userIsSure():
self.output("Deleting branch %r" % (branch, ))
volumePath = self._directory.child(volume)
branchPath = volumePath.child("branches").child(branch)
branchPath.remove()
else:
self.output("Aborting.")
def _userIsSure(self, extraMessage=None):
message = "Are you sure? "
if extraMessage:
message += extraMessage
message += " (y/n): "
sys.stdout.write(message)
sys.stdout.flush()
return raw_input().lower() in ("y", "yes")
def setActiveVolume(self, volume):
self._directory.child("current_volume.json").setContent(
json.dumps(dict(current_volume=volume)))
def volume(self):
currentVolume = self._directory.child("current_volume.json")
if currentVolume.exists():
volume = json.loads(currentVolume.getContent())["current_volume"]
else:
raise UsageError("No active volume: use dvol switch to choose one")
if not self._directory.child(volume).exists():
raise UsageError("Active volume %s does not exist: "
"use dvol switch to choose another" % (volume, ))
return volume
def setActiveBranch(self, volume, branch):
self._directory.child(volume).child("current_branch.json").setContent(
json.dumps(dict(current_branch=branch)))
self.lock.acquire(volume)
try:
self.updateRunningPoint(volume)
finally:
self.lock.release(volume)
def getActiveBranch(self, volume):
currentBranch = self._directory.child(volume).child(
"current_branch.json")
if currentBranch.exists():
return json.loads(currentBranch.getContent())["current_branch"]
else:
return DEFAULT_BRANCH
def updateRunningPoint(self, volume):
"""
construct a stable (wrt switching branches) path with symlinks
"""
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
stablePath = volumePath.child("running_point")
if stablePath.exists():
stablePath.remove()
branchPath.linkTo(stablePath)
return stablePath.path
def commitVolume(self, message):
volume = self.volume()
commitId = (str(uuid.uuid4()) + str(uuid.uuid4())).replace("-",
"")[:40]
self.output(commitId)
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
commitPath = volumePath.child("commits").child(commitId)
if commitPath.exists():
raise Exception("woah, random uuid collision. try again!")
# Make the commits directory if necessary
if not commitPath.parent().exists():
commitPath.parent().makedirs()
# acquire lock (read: stop containers) to ensure consistent snapshot
# with file-copy based backend
# XXX tests for acquire/release
self.lock.acquire(volume)
try:
copyTo(branchPath, commitPath)
finally:
self.lock.release(volume)
self._recordCommit(volume, branchName, commitId, message)
def _recordCommit(self, volume, branch, commitId, message):
commitData = self.commitDatabase.read(volume, branch)
commitData.append(dict(id=commitId, message=message))
self.commitDatabase.write(volume, branch, commitData)
def exists(self, volume):
volumePath = self._directory.child(volume)
return volumePath.exists()
def listVolumes(self):
table = get_table()
table.set_cols_align(["l", "l", "l"])
dc = self.lock.containers # XXX ugly
volumes = [v for v in self._directory.children() if v.isdir()]
activeVolume = None
if volumes:
try:
activeVolume = self.volume()
except UsageError:
# don't refuse to list volumes just because none of them are active
pass
rows = [["", "", ""]] + [[" VOLUME", "BRANCH", "CONTAINERS"]] + [[
("*" if v.basename() == activeVolume else " ") + " " +
v.basename(),
self.getActiveBranch(v.basename()), ",".join(
c['Name'] for c in dc.get_related_containers(v.basename()))
] for v in sorted(volumes)]
table.add_rows(rows)
self.output(table.draw())
def listCommits(self, branch=None):
if branch is None:
branch = self.getActiveBranch(self.volume())
volume = self.volume()
aggregate = []
for commit in reversed(self.commitDatabase.read(volume, branch)):
# TODO fill in author/date
aggregate.append("commit %(id)s\n"
"Author: Who knows <mystery@person>\n"
"Date: Whenever\n"
"\n"
" %(message)s\n" % commit)
self.output("\n".join(aggregate))
def _resolveNamedCommitCurrentBranch(self, commit, volume):
branch = self.getActiveBranch(volume)
remainder = commit[len("HEAD"):]
if remainder == "^" * len(remainder):
offset = len(remainder)
else:
raise UsageError("Malformed commit identifier %r" % (commit, ))
commits = self.commitDatabase.read(volume, branch)
# commits are appended to, so the last one is the latest
return commits[-1 - offset]["id"]
def _destroyNewerCommits(self, commit, volume):
# TODO in the future, we'll care more about the following being an
# atomic operation
branch = self.getActiveBranch(volume)
commits = self.commitDatabase.read(volume, branch)
commitIndex = [c["id"] for c in commits].index(commit) + 1
remainingCommits = commits[:commitIndex]
destroyCommits = commits[commitIndex:]
# look in all branches for commit references before removing them
totalCommits = set()
for otherBranch in self.allBranches(volume):
if otherBranch == branch:
# skip this branch, otherwise we'll never destroy any commits
continue
commits = self.commitDatabase.read(volume, branch)
totalCommits.update(commit["id"] for commit in commits)
for commit in destroyCommits:
commitId = commit["id"]
if commitId in totalCommits:
# skip destroying this commit; it is still actively referred to
# in another branch
continue
volumePath = self._directory.child(volume)
commitPath = volumePath.child("commits").child(commitId)
commitPath.remove()
self.commitDatabase.write(volume, branch, remainingCommits)
def resetVolume(self, commit):
"""
Forcefully roll back the current working copy to this commit,
destroying any later commits.
"""
volume = self.volume()
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
if commit.startswith("HEAD"):
try:
commit = self._resolveNamedCommitCurrentBranch(commit, volume)
except IndexError:
self.output("Referenced commit does not exist; check dvol log")
return
commitPath = volumePath.child("commits").child(commit)
if not commitPath.exists():
raise NoSuchCommit("commit '%s' does not exist" % (commit, ))
self.lock.acquire(volume)
try:
branchPath.remove()
copyTo(commitPath, branchPath)
self._destroyNewerCommits(commit, volume)
finally:
self.lock.release(volume)
def seedVolumes(self, compose_file):
# XXX: does not work with absolute paths, but should
compose = yaml.load(PWD_PATH.child(compose_file).open())
valid_volumes = []
# TODO: will need updating for new compose file format
for service, config in compose.iteritems():
if "volume_driver" in config and config["volume_driver"] == "dvol":
for volume in config["volumes"]:
if ":" in volume and not volume.startswith("/"):
valid_volumes.append((volume, config))
if not valid_volumes:
self.output(
'No volumes found with "volume_driver: dvol" and a named volume (like "volumename:/path_inside_container")! Please check your docker-compose.yml file.'
)
else:
self.output(
"Please seed your dvol volume(s) by running the following command(s):"
)
for volume, config in valid_volumes:
# TODO: need some validation before running commands with string interpolation here, docker-compose file could be malicious
# TODO: would be better if we ran the command for the user, rather than making them copy and paste
self.output(
"docker run --volume-driver=dvol -v %(volume)s:/_target -ti %(image)s sh -c 'cp -av %(source)s/* /_target/'"
% dict(
volume=volume.split(":")[0],
source=volume.split(":")[1],
image=config["image"],
))
verbose = False
global SCHEMADIR, PGSOCKETDIR, PSQL
for opt, arg in optargs:
if opt in ("-h", "--help"):
usage()
elif opt in ("-d",):
SCHEMADIR = arg
elif opt in ("-k",):
PGSOCKETDIR = arg
elif opt in ("-p",):
PSQL = arg
elif opt in ("-x",):
sktdir = FilePath("/var/run/caldavd")
for skt in sktdir.children():
if skt.basename().startswith("ccs_postgres_"):
PGSOCKETDIR = skt.path
PSQL = "/Applications/Server.app/Contents/ServerRoot/usr/bin/psql"
SCHEMADIR = "/Applications/Server.app/Contents/ServerRoot/usr/share/caldavd/lib/python/txdav/common/datastore/sql_schema/"
elif opt in ("-v", "--verbose"):
verbose = True
else:
raise NotImplementedError(opt)
# Retrieve the db_version number of the installed schema
try:
db_version = getSchemaVersion(verbose=verbose)
except CheckSchemaError, e:
db_version = 0
def test_exportMixAndMatch(self):
"""
Run the export with some calendars and some addressbooks
"""
yield populateAddressBooksFrom(
{
"user01": {
"addressbook": {
"1.vcf": adbk1Root.child("1.vcf").getContent(),
"2.vcf": adbk1Root.child("2.vcf").getContent(),
"3.vcf": adbk1Root.child("3.vcf").getContent(),
}
},
"user02": {
"addressbook": {
"1.vcf": adbk1Root.child("1.vcf").getContent(),
},
}
}, self.store)
yield populateCalendarsFrom(
{
"user01": {
"calendar1": {
"valentines-day.ics": (valentines, {}),
"new-years-day.ics": (newYears, {})
}
},
"user02": {
"calendar1": {
"valentines-day.ics": (valentines, {})
},
"calendar2": {
"new-years-day.ics": (newYears, {})
}
}
}, self.store)
outputDir = FilePath(self.mktemp())
outputDir.makedirs()
main([
'calendarserver_export',
'--directory',
outputDir.path,
'--uid',
'user01',
'--contacts',
'--uid',
'user01',
'--calendars',
'--uid',
'user02',
'--collection=calendar1',
'--uid',
'user02',
'--contacts',
],
reactor=self)
yield self.waitToStop
self.assertEquals(
set([
"user01_addressbook.vcf", "user01_calendar1.ics",
"user02_calendar1.ics", "user02_addressbook.vcf"
]), set([child.basename() for child in outputDir.children()]))
class FSItem(BackendItem):
logCategory = 'fs_item'
def __init__(
self,
object_id,
parent,
path,
mimetype,
urlbase,
UPnPClass,
update=False,
store=None,
):
BackendItem.__init__(self)
self.id = object_id
self.parent = parent
if parent:
parent.add_child(self, update=update)
if mimetype == 'root':
self.location = str(path)
else:
if mimetype == 'item' and path is None:
path = os.path.join(parent.get_realpath(), str(self.id))
# self.location = FilePath(unicode(path))
self.location = FilePath(path)
self.mimetype = mimetype
if urlbase[-1] != '/':
urlbase += '/'
self.url = urlbase + str(self.id)
self.store = store
if parent is None:
parent_id = -1
else:
parent_id = parent.get_id()
self.item = UPnPClass(object_id, parent_id, self.get_name())
if isinstance(self.item, Container):
self.item.childCount = 0
self.child_count = 0
self.children = []
self.sorted = False
self.caption = None
if mimetype in ['directory', 'root']:
self.update_id = 0
self.get_url = lambda: self.url
# self.item.searchable = True
# self.item.searchClass = 'object'
if (isinstance(self.location, FilePath)
and self.location.isdir() is True):
self.check_for_cover_art()
if getattr(self, 'cover', None):
_, ext = os.path.splitext(self.cover)
''' add the cover image extension to help clients
not reacting on the mimetype '''
self.item.albumArtURI = ''.join(
(urlbase, str(self.id), '?cover', str(ext)))
else:
self.get_url = lambda: self.url
if self.mimetype.startswith('audio/'):
if getattr(parent, 'cover', None):
_, ext = os.path.splitext(parent.cover)
''' add the cover image extension to help clients
not reacting on the mimetype '''
self.item.albumArtURI = ''.join(
(urlbase, str(self.id), '?cover', ext))
_, host_port, _, _, _ = urlsplit(urlbase)
if host_port.find(':') != -1:
host, port = tuple(host_port.split(':'))
else:
host = host_port
try:
size = self.location.getsize()
except Exception:
size = 0
if (self.store.server and self.store.server.coherence.config.get(
'transcoding', 'no') == 'yes'):
if self.mimetype in (
'application/ogg',
'audio/ogg',
'audio/x-wav',
'audio/x-m4a',
'application/x-flac',
):
new_res = Resource(
self.url + '/transcoded.mp3',
f'http-get:*:{"audio/mpeg"}:*',
)
new_res.size = None
# self.item.res.append(new_res)
if mimetype != 'item':
res = Resource(
'file://' + quote(self.get_path(), encoding='utf-8'),
f'internal:{host}:{self.mimetype}:*',
)
res.size = size
self.item.res.append(res)
if mimetype != 'item':
res = Resource(self.url, f'http-get:*:{self.mimetype}:*')
else:
res = Resource(self.url, 'http-get:*:*:*')
res.size = size
self.item.res.append(res)
''' if this item is of type audio and we want to add a transcoding
rule for it, this is the way to do it:
create a new Resource object, at least a 'http-get'
and maybe an 'internal' one too
for transcoding to wav this looks like that
res = Resource(
url_for_transcoded audio,
'http-get:*:audio/x-wav:%s'% ';'.join(
['DLNA.ORG_PN=JPEG_TN']+simple_dlna_tags))
res.size = None
self.item.res.append(res)
'''
if (self.store.server and self.store.server.coherence.config.get(
'transcoding', 'no') == 'yes'):
if self.mimetype in (
'audio/mpeg',
'application/ogg',
'audio/ogg',
'audio/x-wav',
'audio/x-m4a',
'audio/flac',
'application/x-flac',
):
dlna_pn = 'DLNA.ORG_PN=LPCM'
dlna_tags = simple_dlna_tags[:]
# dlna_tags[1] = 'DLNA.ORG_OP=00'
dlna_tags[2] = 'DLNA.ORG_CI=1'
new_res = Resource(
self.url + '?transcoded=lpcm',
f'http-get:*:{"audio/L16;rate=44100;channels=2"}:'
f'{";".join([dlna_pn] + dlna_tags)}',
)
new_res.size = None
# self.item.res.append(new_res)
if self.mimetype != 'audio/mpeg':
new_res = Resource(
self.url + '?transcoded=mp3',
f'http-get:*:{"audio/mpeg"}:*',
)
new_res.size = None
# self.item.res.append(new_res)
''' if this item is an image and we want to add a thumbnail for it
we have to follow these rules:
create a new Resource object, at least a 'http-get'
and maybe an 'internal' one too
for an JPG this looks like that
res = Resource(url_for_thumbnail,
'http-get:*:image/jpg:%s'% ';'.join(
['DLNA.ORG_PN=JPEG_TN']+simple_dlna_tags))
res.size = size_of_thumbnail
self.item.res.append(res)
and for a PNG the Resource creation is like that
res = Resource(url_for_thumbnail,
'http-get:*:image/png:%s'% ';'.join(
simple_dlna_tags+['DLNA.ORG_PN=PNG_TN']))
if not hasattr(self.item, 'attachments'):
self.item.attachments = {}
self.item.attachments[key] = utils.StaticFile(
filename_of_thumbnail)
'''
if (self.mimetype in ('image/jpeg', 'image/png')
or self.mimetype.startswith('video/')):
try:
filename, mimetype, dlna_pn = _find_thumbnail(
self.get_path())
except NoThumbnailFound:
pass
except Exception:
self.warning(traceback.format_exc())
else:
dlna_tags = simple_dlna_tags[:]
dlna_tags[
3] = 'DLNA.ORG_FLAGS=00f00000000000000000000000000000'
hash_from_path = str(id(filename))
new_res = Resource(
self.url + '?attachment=' + hash_from_path,
f'http-get:*:{mimetype}:'
f'{";".join([dlna_pn] + dlna_tags)}',
)
new_res.size = os.path.getsize(filename)
self.item.res.append(new_res)
if not hasattr(self.item, 'attachments'):
self.item.attachments = {}
self.item.attachments[hash_from_path] = utils.StaticFile(
filename)
if self.mimetype.startswith('video/'):
# check for a subtitles file
caption, _ = os.path.splitext(self.get_path())
caption = caption + '.srt'
if os.path.exists(caption):
hash_from_path = str(id(caption))
mimetype = 'smi/caption'
new_res = Resource(
self.url + '?attachment=' + hash_from_path,
f'http-get:*:{mimetype}:{"*"}',
)
new_res.size = os.path.getsize(caption)
self.caption = new_res.data
self.item.res.append(new_res)
if not hasattr(self.item, 'attachments'):
self.item.attachments = {}
self.item.attachments[hash_from_path] = utils.StaticFile(
caption,
defaultType=mimetype,
)
try:
# FIXME: getmtime is deprecated in Twisted 2.6
self.item.date = datetime.fromtimestamp(
self.location.getmtime())
except Exception:
self.item.date = None
def rebuild(self, urlbase):
# print('rebuild', self.mimetype)
if self.mimetype != 'item':
return
# print('rebuild for', self.get_path())
mimetype, _ = mimetypes.guess_type(self.get_path(), strict=False)
if mimetype is None:
return
self.mimetype = mimetype
# print('rebuild', self.mimetype)
UPnPClass = classChooser(self.mimetype)
self.item = UPnPClass(self.id, self.parent.id, self.get_name())
if getattr(self.parent, 'cover', None):
_, ext = os.path.splitext(self.parent.cover)
# add the cover image extension to help
# clients not reacting on the mimetype
self.item.albumArtURI = ''.join(
(urlbase, str(self.id), '?cover', ext))
_, host_port, _, _, _ = urlsplit(urlbase)
if host_port.find(':') != -1:
host, port = tuple(host_port.split(':'))
else:
host = host_port
res = Resource(
'file://' + quote(self.get_path()),
f'internal:{host}:{self.mimetype}:*',
)
try:
res.size = self.location.getsize()
except Exception:
res.size = 0
self.item.res.append(res)
res = Resource(self.url, f'http-get:*:{self.mimetype}:*')
try:
res.size = self.location.getsize()
except Exception:
res.size = 0
self.item.res.append(res)
try:
# FIXME: getmtime is deprecated in Twisted 2.6
self.item.date = datetime.fromtimestamp(self.location.getmtime())
except Exception:
self.item.date = None
self.parent.update_id += 1
def check_for_cover_art(self):
''' let's try to find in the current directory some jpg file,
or png if the jpg search fails, and take the first one
that comes around
'''
try:
jpgs = [
i.path for i in self.location.children()
if i.splitext()[1] in ('.jpg', '.JPG')
]
try:
self.cover = jpgs[0]
except IndexError:
pngs = [
i.path for i in self.location.children()
if i.splitext()[1] in ('.png', '.PNG')
]
try:
self.cover = pngs[0]
except IndexError:
return
except UnicodeDecodeError:
self.warning(
f'UnicodeDecodeError - there is something wrong with a ' +
f'file located in {self.location.path}')
def remove(self):
# print('FSItem remove', self.id, self.get_name(), self.parent)
if self.parent:
self.parent.remove_child(self)
del self.item
def add_child(self, child, update=False):
self.children.append(child)
self.child_count += 1
if isinstance(self.item, Container):
self.item.childCount += 1
if update:
self.update_id += 1
self.sorted = False
def remove_child(self, child):
# print(f'remove_from {self.id:d} ({self.get_name()}) '
# f'child {child.id:d} ({child.get_name()})')
if child in self.children:
self.child_count -= 1
if isinstance(self.item, Container):
self.item.childCount -= 1
self.children.remove(child)
self.update_id += 1
self.sorted = False
def get_children(self, start=0, request_count=0):
if not self.sorted:
self.children.sort(key=_natural_key)
self.sorted = True
if request_count == 0:
return self.children[start:]
else:
return self.children[start:request_count]
def get_child_count(self):
return self.child_count
def get_id(self):
return self.id
def get_update_id(self):
if hasattr(self, 'update_id'):
return self.update_id
else:
return None
def get_path(self):
if self.mimetype in ['directory', 'root']:
return None
if isinstance(self.location, FilePath):
return self.location.path
else:
return self.location
def get_realpath(self):
if isinstance(self.location, FilePath):
return self.location.path
else:
return self.location
def set_path(self, path=None, extension=None):
if path is None:
path = self.get_path()
if extension is not None:
path, old_ext = os.path.splitext(path)
path = ''.join((path, extension))
if isinstance(self.location, FilePath):
self.location = FilePath(path)
else:
self.location = path
def get_name(self):
if isinstance(self.location, FilePath):
name = self.location.basename()
else:
name = self.location
return name
def get_cover(self):
if self.cover:
return self.cover
try:
return self.parent.cover
except AttributeError:
return None
def get_parent(self):
return self.parent
def get_item(self):
return self.item
def get_xml(self):
return self.item.toString()
def __repr__(self):
return ('id: ' + str(self.id) + ' @ ' +
str(self.get_name().encode('ascii', 'xmlcharrefreplace')))
class FSItem(BackendItem):
logCategory = 'fs_item'
def __init__(self, object_id, parent, path, mimetype, urlbase, UPnPClass,update=False):
self.id = object_id
self.parent = parent
if parent:
parent.add_child(self,update=update)
if mimetype == 'root':
self.location = unicode(path)
else:
if mimetype == 'item' and path is None:
path = os.path.join(parent.get_path(),unicode(self.id))
#self.location = FilePath(unicode(path))
self.location = FilePath(path)
self.mimetype = mimetype
if urlbase[-1] != '/':
urlbase += '/'
self.url = urlbase + str(self.id)
if parent == None:
parent_id = -1
else:
parent_id = parent.get_id()
self.item = UPnPClass(object_id, parent_id, self.get_name())
if isinstance(self.item, Container):
self.item.childCount = 0
self.child_count = 0
self.children = []
if mimetype in ['directory','root']:
self.update_id = 0
self.get_url = lambda : self.url
self.get_path = lambda : None
#self.item.searchable = True
#self.item.searchClass = 'object'
if(isinstance(self.location,FilePath) and
self.location.isdir() == True):
self.check_for_cover_art()
if hasattr(self, 'cover'):
_,ext = os.path.splitext(self.cover)
""" add the cover image extension to help clients not reacting on
the mimetype """
self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))
else:
self.get_url = lambda : self.url
if self.mimetype.startswith('audio/'):
if hasattr(parent, 'cover'):
_,ext = os.path.splitext(parent.cover)
""" add the cover image extension to help clients not reacting on
the mimetype """
self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))
_,host_port,_,_,_ = urlsplit(urlbase)
if host_port.find(':') != -1:
host,port = tuple(host_port.split(':'))
else:
host = host_port
try:
size = self.location.getsize()
except:
size = 0
if mimetype != 'item':
res = Resource('file://'+ urllib.quote(self.get_path()), 'internal:%s:%s:*' % (host,self.mimetype))
res.size = size
self.item.res.append(res)
if mimetype != 'item':
res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)
else:
res = Resource(self.url, 'http-get:*:*:*')
res.size = size
self.item.res.append(res)
""" if this item is an image and we want to add a thumbnail for it
we have to follow these rules:
create a new Resource object, at least a 'http-get'
and maybe an 'internal' one too
for an JPG this looks like that
res = Resource(url_for_thumbnail,
'http-get:*:image/jpg:%s'% ';'.join(simple_dlna_tags+('DLNA.ORG_PN=JPEG_TN',)))
res.size = size_of_thumbnail
self.item.res.append(res)
and for a PNG the Resource creation is like that
res = Resource(url_for_thumbnail,
'http-get:*:image/png:%s'% ';'.join(simple_dlna_tags+('DLNA.ORG_PN=PNG_TN',)))
if not hasattr(self.item, 'attachments'):
self.item.attachments = {}
self.item.attachments[key] = utils.StaticFile(filename_of_thumbnail)
"""
if self.mimetype in ('image/jpeg', 'image/png'):
path = self.get_path()
thumbnail = os.path.join(os.path.dirname(path),'.thumbs',os.path.basename(path))
if os.path.exists(thumbnail):
mimetype,_ = mimetypes.guess_type(thumbnail, strict=False)
if mimetype in ('image/jpeg','image/png'):
if mimetype == 'image/jpeg':
dlna_pn = 'DLNA.ORG_PN=JPEG_TN'
else:
dlna_pn = 'DLNA.ORG_PN=PNG_TN'
hash_from_path = str(id(thumbnail))
new_res = Resource(self.url+'?attachment='+hash_from_path,
'http-get:*:%s:%s' % (mimetype, ';'.join(simple_dlna_tags+(dlna_pn,))))
new_res.size = os.path.getsize(thumbnail)
self.item.res.append(new_res)
if not hasattr(self.item, 'attachments'):
self.item.attachments = {}
self.item.attachments[hash_from_path] = utils.StaticFile(urllib.quote(thumbnail))
try:
# FIXME: getmtime is deprecated in Twisted 2.6
self.item.date = datetime.fromtimestamp(self.location.getmtime())
except:
self.item.date = None
def rebuild(self, urlbase):
#print "rebuild", self.mimetype
if self.mimetype != 'item':
return
#print "rebuild for", self.get_path()
mimetype,_ = mimetypes.guess_type(self.get_path(),strict=False)
if mimetype == None:
return
self.mimetype = mimetype
#print "rebuild", self.mimetype
UPnPClass = classChooser(self.mimetype)
self.item = UPnPClass(self.id, self.parent.id, self.get_name())
if hasattr(self.parent, 'cover'):
_,ext = os.path.splitext(self.parent.cover)
""" add the cover image extension to help clients not reacting on
the mimetype """
self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))
_,host_port,_,_,_ = urlsplit(urlbase)
if host_port.find(':') != -1:
host,port = tuple(host_port.split(':'))
else:
host = host_port
res = Resource('file://'+urllib.quote(self.get_path()), 'internal:%s:%s:*' % (host,self.mimetype))
try:
res.size = self.location.getsize()
except:
res.size = 0
self.item.res.append(res)
res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)
try:
res.size = self.location.getsize()
except:
res.size = 0
self.item.res.append(res)
try:
# FIXME: getmtime is deprecated in Twisted 2.6
self.item.date = datetime.fromtimestamp(self.location.getmtime())
except:
self.item.date = None
self.parent.update_id += 1
def check_for_cover_art(self):
""" let's try to find in the current directory some jpg file,
or png if the jpg search fails, and take the first one
that comes around
"""
try:
jpgs = [i.path for i in self.location.children() if i.splitext()[1] in ('.jpg', '.JPG')]
try:
self.cover = jpgs[0]
except IndexError:
pngs = [i.path for i in self.location.children() if i.splitext()[1] in ('.png', '.PNG')]
try:
self.cover = pngs[0]
except IndexError:
return
except UnicodeDecodeError:
self.warning("UnicodeDecodeError - there is something wrong with a file located in %r", self.location.path)
def remove(self):
#print "FSItem remove", self.id, self.get_name(), self.parent
if self.parent:
self.parent.remove_child(self)
del self.item
def add_child(self, child, update=False):
self.children.append(child)
self.child_count += 1
if isinstance(self.item, Container):
self.item.childCount += 1
if update == True:
self.update_id += 1
def remove_child(self, child):
#print "remove_from %d (%s) child %d (%s)" % (self.id, self.get_name(), child.id, child.get_name())
if child in self.children:
self.child_count -= 1
if isinstance(self.item, Container):
self.item.childCount -= 1
self.children.remove(child)
self.update_id += 1
def get_children(self,start=0,request_count=0):
if request_count == 0:
return self.children[start:]
else:
return self.children[start:request_count]
def get_child_count(self):
return self.child_count
def get_id(self):
return self.id
def get_update_id(self):
if hasattr(self, 'update_id'):
return self.update_id
else:
return None
def get_path(self):
if isinstance( self.location,FilePath):
return self.location.path
else:
self.location
def set_path(self,path=None,extension=None):
if path is None:
path = self.get_path()
if extension is not None:
path,old_ext = os.path.splitext(path)
path = ''.join((path,extension))
if isinstance( self.location,FilePath):
self.location = FilePath(path)
else:
self.location = path
def get_name(self):
if isinstance( self.location,FilePath):
name = self.location.basename().decode("utf-8", "replace")
else:
name = self.location.decode("utf-8", "replace")
return name
def get_cover(self):
try:
return self.cover
except:
try:
return self.parent.cover
except:
return ''
def get_parent(self):
return self.parent
def get_item(self):
return self.item
def get_xml(self):
return self.item.toString()
def __repr__(self):
return 'id: ' + str(self.id) + ' @ ' + self.get_name().encode('ascii','xmlcharrefreplace')
class NewsBuilderTests(TestCase, StructureAssertingMixin):
"""
Tests for L{NewsBuilder}.
"""
skip = svnSkip
def setUp(self):
"""
Create a fake project and stuff some basic structure and content into
it.
"""
self.builder = NewsBuilder()
self.project = FilePath(self.mktemp())
self.project.createDirectory()
self.existingText = 'Here is stuff which was present previously.\n'
self.createStructure(
self.project, {
'NEWS': self.existingText,
'5.feature': 'We now support the web.\n',
'12.feature': 'The widget is more robust.\n',
'15.feature': (
'A very long feature which takes many words to '
'describe with any accuracy was introduced so that '
'the line wrapping behavior of the news generating '
'code could be verified.\n'),
'16.feature': (
'A simpler feature\ndescribed on multiple lines\n'
'was added.\n'),
'23.bugfix': 'Broken stuff was fixed.\n',
'25.removal': 'Stupid stuff was deprecated.\n',
'30.misc': '',
'35.misc': '',
'40.doc': 'foo.bar.Baz.quux',
'41.doc': 'writing Foo servers'})
def svnCommit(self, project=None):
"""
Make the C{project} directory a valid subversion directory with all
files committed.
"""
if project is None:
project = self.project
repositoryPath = self.mktemp()
repository = FilePath(repositoryPath)
runCommand(["svnadmin", "create", repository.path])
runCommand(["svn", "checkout", "file://" + repository.path,
project.path])
runCommand(["svn", "add"] + glob.glob(project.path + "/*"))
runCommand(["svn", "commit", project.path, "-m", "yay"])
def test_today(self):
"""
L{NewsBuilder._today} returns today's date in YYYY-MM-DD form.
"""
self.assertEqual(
self.builder._today(), date.today().strftime('%Y-%m-%d'))
def test_findFeatures(self):
"""
When called with L{NewsBuilder._FEATURE}, L{NewsBuilder._findChanges}
returns a list of bugfix ticket numbers and descriptions as a list of
two-tuples.
"""
features = self.builder._findChanges(
self.project, self.builder._FEATURE)
self.assertEqual(
features,
[(5, "We now support the web."),
(12, "The widget is more robust."),
(15,
"A very long feature which takes many words to describe with "
"any accuracy was introduced so that the line wrapping behavior "
"of the news generating code could be verified."),
(16, "A simpler feature described on multiple lines was added.")])
def test_findBugfixes(self):
"""
When called with L{NewsBuilder._BUGFIX}, L{NewsBuilder._findChanges}
returns a list of bugfix ticket numbers and descriptions as a list of
two-tuples.
"""
bugfixes = self.builder._findChanges(
self.project, self.builder._BUGFIX)
self.assertEqual(
bugfixes,
[(23, 'Broken stuff was fixed.')])
def test_findRemovals(self):
"""
When called with L{NewsBuilder._REMOVAL}, L{NewsBuilder._findChanges}
returns a list of removal/deprecation ticket numbers and descriptions
as a list of two-tuples.
"""
removals = self.builder._findChanges(
self.project, self.builder._REMOVAL)
self.assertEqual(
removals,
[(25, 'Stupid stuff was deprecated.')])
def test_findDocumentation(self):
"""
When called with L{NewsBuilder._DOC}, L{NewsBuilder._findChanges}
returns a list of documentation ticket numbers and descriptions as a
list of two-tuples.
"""
doc = self.builder._findChanges(
self.project, self.builder._DOC)
self.assertEqual(
doc,
[(40, 'foo.bar.Baz.quux'),
(41, 'writing Foo servers')])
def test_findMiscellaneous(self):
"""
When called with L{NewsBuilder._MISC}, L{NewsBuilder._findChanges}
returns a list of removal/deprecation ticket numbers and descriptions
as a list of two-tuples.
"""
misc = self.builder._findChanges(
self.project, self.builder._MISC)
self.assertEqual(
misc,
[(30, ''),
(35, '')])
def test_writeHeader(self):
"""
L{NewsBuilder._writeHeader} accepts a file-like object opened for
writing and a header string and writes out a news file header to it.
"""
output = StringIO()
self.builder._writeHeader(output, "Super Awesometastic 32.16")
self.assertEqual(
output.getvalue(),
"Super Awesometastic 32.16\n"
"=========================\n"
"\n")
def test_writeSection(self):
"""
L{NewsBuilder._writeSection} accepts a file-like object opened for
writing, a section name, and a list of ticket information (as returned
by L{NewsBuilder._findChanges}) and writes out a section header and all
of the given ticket information.
"""
output = StringIO()
self.builder._writeSection(
output, "Features",
[(3, "Great stuff."),
(17, "Very long line which goes on and on and on, seemingly "
"without end until suddenly without warning it does end.")])
self.assertEqual(
output.getvalue(),
"Features\n"
"--------\n"
" - Great stuff. (#3)\n"
" - Very long line which goes on and on and on, seemingly "
"without end\n"
" until suddenly without warning it does end. (#17)\n"
"\n")
def test_writeMisc(self):
"""
L{NewsBuilder._writeMisc} accepts a file-like object opened for
writing, a section name, and a list of ticket information (as returned
by L{NewsBuilder._findChanges} and writes out a section header and all
of the ticket numbers, but excludes any descriptions.
"""
output = StringIO()
self.builder._writeMisc(
output, "Other",
[(x, "") for x in range(2, 50, 3)])
self.assertEqual(
output.getvalue(),
"Other\n"
"-----\n"
" - #2, #5, #8, #11, #14, #17, #20, #23, #26, #29, #32, #35, "
"#38, #41,\n"
" #44, #47\n"
"\n")
def test_build(self):
"""
L{NewsBuilder.build} updates a NEWS file with new features based on the
I{<ticket>.feature} files found in the directory specified.
"""
self.builder.build(
self.project, self.project.child('NEWS'),
"Super Awesometastic 32.16")
results = self.project.child('NEWS').getContent()
self.assertEqual(
results,
'Super Awesometastic 32.16\n'
'=========================\n'
'\n'
'Features\n'
'--------\n'
' - We now support the web. (#5)\n'
' - The widget is more robust. (#12)\n'
' - A very long feature which takes many words to describe '
'with any\n'
' accuracy was introduced so that the line wrapping behavior '
'of the\n'
' news generating code could be verified. (#15)\n'
' - A simpler feature described on multiple lines was '
'added. (#16)\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Improved Documentation\n'
'----------------------\n'
' - foo.bar.Baz.quux (#40)\n'
' - writing Foo servers (#41)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n'
'Other\n'
'-----\n'
' - #30, #35\n'
'\n\n' + self.existingText)
def test_emptyProjectCalledOut(self):
"""
If no changes exist for a project, I{NEWS} gains a new section for
that project that includes some helpful text about how there were no
interesting changes.
"""
project = FilePath(self.mktemp()).child("twisted")
project.makedirs()
self.createStructure(project, {'NEWS': self.existingText})
self.builder.build(
project, project.child('NEWS'),
"Super Awesometastic 32.16")
results = project.child('NEWS').getContent()
self.assertEqual(
results,
'Super Awesometastic 32.16\n'
'=========================\n'
'\n' +
self.builder._NO_CHANGES +
'\n\n' + self.existingText)
def test_preserveTicketHint(self):
"""
If a I{NEWS} file begins with the two magic lines which point readers
at the issue tracker, those lines are kept at the top of the new file.
"""
news = self.project.child('NEWS')
news.setContent(
'Ticket numbers in this file can be looked up by visiting\n'
'http://twistedmatrix.com/trac/ticket/<number>\n'
'\n'
'Blah blah other stuff.\n')
self.builder.build(self.project, news, "Super Awesometastic 32.16")
self.assertEqual(
news.getContent(),
'Ticket numbers in this file can be looked up by visiting\n'
'http://twistedmatrix.com/trac/ticket/<number>\n'
'\n'
'Super Awesometastic 32.16\n'
'=========================\n'
'\n'
'Features\n'
'--------\n'
' - We now support the web. (#5)\n'
' - The widget is more robust. (#12)\n'
' - A very long feature which takes many words to describe '
'with any\n'
' accuracy was introduced so that the line wrapping behavior '
'of the\n'
' news generating code could be verified. (#15)\n'
' - A simpler feature described on multiple lines was '
'added. (#16)\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Improved Documentation\n'
'----------------------\n'
' - foo.bar.Baz.quux (#40)\n'
' - writing Foo servers (#41)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n'
'Other\n'
'-----\n'
' - #30, #35\n'
'\n\n'
'Blah blah other stuff.\n')
def test_emptySectionsOmitted(self):
"""
If there are no changes of a particular type (feature, bugfix, etc), no
section for that type is written by L{NewsBuilder.build}.
"""
for ticket in self.project.children():
if ticket.splitext()[1] in ('.feature', '.misc', '.doc'):
ticket.remove()
self.builder.build(
self.project, self.project.child('NEWS'),
'Some Thing 1.2')
self.assertEqual(
self.project.child('NEWS').getContent(),
'Some Thing 1.2\n'
'==============\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n\n'
'Here is stuff which was present previously.\n')
def test_duplicatesMerged(self):
"""
If two change files have the same contents, they are merged in the
generated news entry.
"""
def feature(s):
return self.project.child(s + '.feature')
feature('5').copyTo(feature('15'))
feature('5').copyTo(feature('16'))
self.builder.build(
self.project, self.project.child('NEWS'),
'Project Name 5.0')
self.assertEqual(
self.project.child('NEWS').getContent(),
'Project Name 5.0\n'
'================\n'
'\n'
'Features\n'
'--------\n'
' - We now support the web. (#5, #15, #16)\n'
' - The widget is more robust. (#12)\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Improved Documentation\n'
'----------------------\n'
' - foo.bar.Baz.quux (#40)\n'
' - writing Foo servers (#41)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n'
'Other\n'
'-----\n'
' - #30, #35\n'
'\n\n'
'Here is stuff which was present previously.\n')
def createFakeTwistedProject(self):
"""
Create a fake-looking Twisted project to build from.
"""
project = FilePath(self.mktemp()).child("twisted")
project.makedirs()
self.createStructure(
project, {
'NEWS': 'Old boring stuff from the past.\n',
'_version.py': genVersion("twisted", 1, 2, 3),
'topfiles': {
'NEWS': 'Old core news.\n',
'3.feature': 'Third feature addition.\n',
'5.misc': ''},
'conch': {
'_version.py': genVersion("twisted.conch", 3, 4, 5),
'topfiles': {
'NEWS': 'Old conch news.\n',
'7.bugfix': 'Fixed that bug.\n'}}})
return project
def test_buildAll(self):
"""
L{NewsBuilder.buildAll} calls L{NewsBuilder.build} once for each
subproject, passing that subproject's I{topfiles} directory as C{path},
the I{NEWS} file in that directory as C{output}, and the subproject's
name as C{header}, and then again for each subproject with the
top-level I{NEWS} file for C{output}. Blacklisted subprojects are
skipped.
"""
builds = []
builder = NewsBuilder()
builder.build = lambda path, output, header: builds.append((
path, output, header))
builder._today = lambda: '2009-12-01'
project = self.createFakeTwistedProject()
self.svnCommit(project)
builder.buildAll(project)
coreTopfiles = project.child("topfiles")
coreNews = coreTopfiles.child("NEWS")
coreHeader = "Twisted Core 1.2.3 (2009-12-01)"
conchTopfiles = project.child("conch").child("topfiles")
conchNews = conchTopfiles.child("NEWS")
conchHeader = "Twisted Conch 3.4.5 (2009-12-01)"
aggregateNews = project.child("NEWS")
self.assertEqual(
builds,
[(conchTopfiles, conchNews, conchHeader),
(conchTopfiles, aggregateNews, conchHeader),
(coreTopfiles, coreNews, coreHeader),
(coreTopfiles, aggregateNews, coreHeader)])
def test_buildAllAggregate(self):
"""
L{NewsBuilder.buildAll} aggregates I{NEWS} information into the top
files, only deleting fragments once it's done.
"""
builder = NewsBuilder()
project = self.createFakeTwistedProject()
self.svnCommit(project)
builder.buildAll(project)
aggregateNews = project.child("NEWS")
aggregateContent = aggregateNews.getContent()
self.assertIn("Third feature addition", aggregateContent)
self.assertIn("Fixed that bug", aggregateContent)
self.assertIn("Old boring stuff from the past", aggregateContent)
def test_changeVersionInNews(self):
"""
L{NewsBuilder._changeVersions} gets the release date for a given
version of a project as a string.
"""
builder = NewsBuilder()
builder._today = lambda: '2009-12-01'
project = self.createFakeTwistedProject()
self.svnCommit(project)
builder.buildAll(project)
newVersion = Version('TEMPLATE', 7, 7, 14)
coreNews = project.child('topfiles').child('NEWS')
# twisted 1.2.3 is the old version.
builder._changeNewsVersion(
coreNews, "Core", Version("twisted", 1, 2, 3),
newVersion, '2010-01-01')
expectedCore = (
'Twisted Core 7.7.14 (2010-01-01)\n'
'================================\n'
'\n'
'Features\n'
'--------\n'
' - Third feature addition. (#3)\n'
'\n'
'Other\n'
'-----\n'
' - #5\n\n\n')
self.assertEqual(
expectedCore + 'Old core news.\n', coreNews.getContent())
def test_removeNEWSfragments(self):
"""
L{NewsBuilder.buildALL} removes all the NEWS fragments after the build
process, using the C{svn} C{rm} command.
"""
builder = NewsBuilder()
project = self.createFakeTwistedProject()
self.svnCommit(project)
builder.buildAll(project)
self.assertEqual(5, len(project.children()))
output = runCommand(["svn", "status", project.path])
removed = [line for line in output.splitlines()
if line.startswith("D ")]
self.assertEqual(3, len(removed))
def test_checkSVN(self):
"""
L{NewsBuilder.buildAll} raises L{NotWorkingDirectory} when the given
path is not a SVN checkout.
"""
self.assertRaises(
NotWorkingDirectory, self.builder.buildAll, self.project)
class FileUploadResource(Resource):
"""
Twisted Web resource that handles file uploads over `HTTP/POST` requests.
"""
log = make_logger()
def __init__(self,
upload_directory,
temp_directory,
form_fields,
upload_session,
options=None):
"""
:param upload_directory: The target directory where uploaded files will be stored.
:type upload_directory: str
:param temp_directory: A temporary directory where chunks of a file being uploaded are stored.
:type temp_directory: str
:param form_fields: Names of HTML form fields used for uploading.
:type form_fields: dict
:param upload_session: An instance of `ApplicationSession` used for publishing progress events.
:type upload_session: obj
:param options: Options for file upload.
:type options: dict or None
"""
Resource.__init__(self)
self._uploadRoot = FilePath(upload_directory)
self._tempDirRoot = FilePath(temp_directory)
self._form_fields = form_fields
self._fileupload_session = upload_session
self._options = options or {}
self._max_file_size = self._options.get('max_file_size',
10 * 1024 * 1024)
self._fileTypes = self._options.get('file_types', None)
self._file_permissions = self._options.get('file_permissions', None)
# track uploaded files / chunks
self._uploads = {}
self.log.info('Upload Resource started.')
# scan the temp dir for uploaded chunks and fill the _uploads dict with it
# so existing uploads can be resumed
# all other remains will be purged
for fileTempDir in self._tempDirRoot.children():
fileTempName = fileTempDir.basename()
if fileTempDir.isdir():
self._uploads[fileTempName] = {
'chunk_list': [],
'origin': 'startup'
}
for chunk in fileTempDir.listdir():
if chunk[:6] == 'chunk_':
self._uploads[fileTempName]['chunk_list'].append(
int(chunk[6:]))
else:
fileTempDir.child(chunk).remove()
# if no chunks detected then remove remains completely
if len(self._uploads[fileTempName]['chunk_list']) == 0:
fileTempDir.remove()
self._uploads.pop(fileTempName, None)
else: # fileTempDir is a file remaining from a single chunk upload
fileTempDir.remove()
self.log.debug("Scanned pending uploads: {uploads}",
uploads=self._uploads)
def render_POST(self, request):
headers = {
x.decode('iso-8859-1'): y.decode('iso-8859-1')
for x, y in request.getAllHeaders().items()
}
origin = headers['host']
postFields = cgi.FieldStorage(fp=request.content,
headers=headers,
environ={"REQUEST_METHOD": "POST"})
f = self._form_fields
filename = postFields[f['file_name']].value
totalSize = int(postFields[f['total_size']].value)
totalChunks = int(postFields[f['total_chunks']].value)
chunkSize = int(postFields[f['chunk_size']].value)
chunkNumber = int(postFields[f['chunk_number']].value)
fileContent = postFields[f['content']].value
if 'chunk_extra' in f and f['chunk_extra'] in postFields:
chunk_extra = json.loads(postFields[f['chunk_extra']].value)
else:
chunk_extra = {}
if 'finish_extra' in f and f['finish_extra'] in postFields:
finish_extra = json.loads(postFields[f['finish_extra']].value)
else:
finish_extra = {}
fileId = filename
# # prepare user specific upload areas
# # NOT YET IMPLEMENTED
# #
# if 'auth_id' in f and f['auth_id'] in postFields:
# auth_id = postFields[f['auth_id']].value
# mydir = os.path.join(self._uploadRoot, auth_id)
# my_temp_dir = os.path.join(self._tempDirRoot, auth_id)
#
# # check if auth_id is a valid directory_name
# #
# if auth_id != auth_id.encode('ascii', 'ignore'):
# msg = "The requestor auth_id must be an ascii string."
# if self._debug:
# log.msg(msg)
# # 415 Unsupported Media Type
# request.setResponseCode(415, msg)
# return msg
# else:
# auth_id = 'anonymous'
# create user specific folder
# mydir = self._uploadRoot
# my_temp_dir = self._tempDirRoot
# if not os.path.exists(mydir):
# os.makedirs(mydir)
# if not os.path.exists(my_temp_dir):
# os.makedirs(my_temp_dir)
# prepare the on_progress publisher
if 'on_progress' in f and f['on_progress'] in postFields and self._fileupload_session != {}:
topic = postFields[f['on_progress']].value
if 'session' in f and f['session'] in postFields:
session = int(postFields[f['session']].value)
publish_options = PublishOptions(eligible=[session])
else:
publish_options = None
def fileupload_publish(payload):
self._fileupload_session.publish(topic,
payload,
options=publish_options)
else:
def fileupload_publish(payload):
pass
# Register upload right at the start to avoid overlapping upload conflicts
#
if fileId not in self._uploads:
self._uploads[fileId] = {'chunk_list': [], 'origin': origin}
chunk_is_first = True
self.log.debug(
'Started upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}',
file_name=fileId,
total_size=totalSize,
total_chunks=totalChunks,
chunk_size=chunkSize,
chunk_number=chunkNumber)
else:
chunk_is_first = False
# If the chunks are read at startup of crossbar any client may claim and resume the pending upload !
#
upl = self._uploads[fileId]
if upl['origin'] == 'startup':
self.log.debug(
'Will try to resume upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}',
file_name=fileId,
total_size=totalSize,
total_chunks=totalChunks,
chunk_size=chunkSize,
chunk_number=chunkNumber)
upl['origin'] = origin
else:
# check if another session is uploading this file already
#
if upl['origin'] != origin:
msg = "File being uploaded is already uploaded in a different session."
self.log.debug(msg)
# 409 Conflict
request.setResponseCode(409, msg.encode('utf8'))
return msg.encode('utf8')
else:
# check if the chunk is being uploaded in this very session already
# this should never happen !
if chunkNumber in upl['chunk_list']:
msg = "Chunk beeing uploaded is already uploading."
self.log.debug(msg)
# Don't throw a conflict. This may be a wanted behaviour.
# Even if an upload would be resumable, you don't have to resume.
# 409 Conflict
# request.setResponseCode(409, msg.encode('utf8'))
# return msg.encode('utf8')
# check file size
#
if totalSize > self._max_file_size:
msg = "Size {} of file to be uploaded exceeds maximum {}".format(
totalSize, self._max_file_size)
self.log.debug(msg)
# 413 Request Entity Too Large
request.setResponseCode(413, msg.encode('utf8'))
return msg.encode('utf8')
# check file extensions
#
extension = os.path.splitext(filename)[1]
if self._fileTypes and extension not in self._fileTypes:
msg = "Type '{}' of file to be uploaded is in allowed types {}".format(
extension, self._fileTypes)
self.log.debug(msg)
# 415 Unsupported Media Type
request.setResponseCode(415, msg.encode('utf8'))
return msg.encode('utf8')
# TODO: check mime type
#
fileTempDir = self._tempDirRoot.child(fileId)
chunkName = fileTempDir.child('chunk_' + str(chunkNumber))
_chunkName = fileTempDir.child('#kfhf3kz412uru578e38viokbjhfvz4w__' +
'chunk_' + str(chunkNumber))
def mergeFile():
# every chunk has to check if it is the last chunk written, except in a single chunk scenario
if totalChunks > 1 and len(
self._uploads[fileId]['chunk_list']) >= totalChunks:
# last chunk
self.log.debug(
'Finished file upload after chunk {chunk_number} with chunk_list {chunk_list}',
chunk_number=chunkNumber,
chunk_list=self._uploads)
# Merge all files into one file and remove the temp files
# TODO: How to avoid the extra file IO ?
finalFileName = self._uploadRoot.child(fileId)
_finalFileName = fileTempDir.child(
'#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName.path, 'wb') as _finalFile:
for cn in range(1, totalChunks + 1):
with open(
fileTempDir.child('chunk_' + str(cn)).path,
'rb') as ff:
_finalFile.write(ff.read())
_finalFileName.moveTo(finalFileName)
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
finalFileName.chmod(perm)
except Exception as e:
msg = "file upload resource - could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
self._uploads.pop(fileId, None)
request.setResponseCode(500, msg.encode('utf8'))
return msg.encode('utf8')
else:
self.log.debug(
"Changed permissions on {file_name} to {permissions}",
file_name=finalFileName,
permissions=self._file_permissions)
# remove the file temp folder
fileTempDir.remove()
self._uploads.pop(fileId, None)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.,
"finish_extra": finish_extra,
"chunk_extra": chunk_extra
})
if chunk_is_first:
# first chunk of file
# publish file upload start
#
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": totalSize,
"status": "started",
"progress": 0.,
"chunk_extra": chunk_extra
})
if totalChunks == 1:
# only one chunk overall -> write file directly
finalFileName = self._uploadRoot.child(fileId)
_finalFileName = self._tempDirRoot.child(
'#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName.path, 'wb') as _finalFile:
_finalFile.write(fileContent)
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
_finalFileName.chmod(perm)
except Exception as e:
# finalFileName.remove()
msg = "Could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
request.setResponseCode(500, msg.encode('utf8'))
return msg.encode('utf8')
else:
self.log.debug(
"Changed permissions on {file_name} to {permissions}",
file_name=finalFileName,
permissions=self._file_permissions)
_finalFileName.moveTo(finalFileName)
if chunkNumber not in self._uploads[fileId]['chunk_list']:
self._uploads[fileId]['chunk_list'].append(chunkNumber)
self._uploads.pop(fileId, None)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.,
"finish_extra": finish_extra,
"chunk_extra": chunk_extra
})
else:
# first of more chunks
# fileTempDir.remove() # any potential conflict should have been resolved above. This should not be necessary!
if not os.path.isdir(fileTempDir.path):
fileTempDir.makedirs()
with open(_chunkName.path, 'wb') as chunk:
chunk.write(fileContent)
_chunkName.moveTo(chunkName) # atomic file system operation
self.log.debug('chunk_' + str(chunkNumber) +
' written and moved to ' + chunkName.path)
# publish file upload progress
#
fileupload_publish({
"id":
fileId,
"chunk":
chunkNumber,
"name":
filename,
"total":
totalSize,
"remaining":
totalSize - chunkSize,
"status":
"progress",
"progress":
round(float(chunkSize) / float(totalSize), 3),
"chunk_extra":
chunk_extra
})
if chunkNumber not in self._uploads[fileId]['chunk_list']:
self._uploads[fileId]['chunk_list'].append(chunkNumber)
mergeFile()
# clean the temp dir once per file upload
self._remove_stale_uploads()
else:
# intermediate chunk
if not os.path.isdir(fileTempDir.path):
fileTempDir.makedirs()
with open(_chunkName.path, 'wb') as chunk:
chunk.write(fileContent)
_chunkName.moveTo(chunkName)
self.log.debug('chunk_' + str(chunkNumber) +
' written and moved to ' + chunkName.path)
if chunkNumber not in self._uploads[fileId]['chunk_list']:
self._uploads[fileId]['chunk_list'].append(chunkNumber)
received = sum(
fileTempDir.child(f).getsize() for f in fileTempDir.listdir())
fileupload_publish({
"id":
fileId,
"chunk":
chunkNumber,
"name":
filename,
"total":
totalSize,
"remaining":
totalSize - received,
"status":
"progress",
"progress":
round(float(received) / float(totalSize), 3),
"chunk_extra":
chunk_extra
})
mergeFile()
# no errors encountered -> respond success
request.setResponseCode(200)
return b''
def _remove_stale_uploads(self):
"""
This only works if there is a temp folder exclusive for crossbar file uploads
if the system temp folder is used then crossbar creates a "crossbar-uploads" there and
uses that as the temp folder for uploads
If you don't clean up regularly an attacker could fill up the OS file system
"""
for fileTempDir in self._tempDirRoot.children():
self.log.debug('REMOVE STALE UPLOADS ' +
str(fileTempDir.basename()))
if fileTempDir.isdir() and (
fileTempDir.basename()) not in self._uploads:
fileTempDir.remove()
def render_GET(self, request):
"""
This method can be used to check whether a chunk has been uploaded already.
It returns with HTTP status code `200` if yes and `404` if not.
The request needs to contain the file identifier and the chunk number to check for.
"""
for param in ['file_name', 'chunk_number']:
if not self._form_fields[param].encode(
'iso-8859-1') in request.args:
msg = "file upload resource - missing request query parameter '{}', configured from '{}'".format(
self._form_fields[param], param)
self.log.debug(msg)
# 400 Bad Request
request.setResponseCode(400, msg.encode('utf8'))
return msg.encode('utf8')
file_name = request.args[self._form_fields['file_name'].encode(
'iso-8859-1')][0].decode('utf8')
chunk_number = int(
request.args[self._form_fields['chunk_number'].encode(
'iso-8859-1')][0].decode('utf8'))
# a complete upload will be repeated an incomplete upload will be resumed
if file_name in self._uploads and chunk_number in self._uploads[
file_name]['chunk_list']:
self.log.debug(
"Skipping chunk upload {file_name} of chunk {chunk_number}",
file_name=file_name,
chunk_number=chunk_number)
msg = b"chunk of file already uploaded"
request.setResponseCode(200, msg)
return msg
else:
msg = b"chunk of file not yet uploaded"
request.setResponseCode(404, msg)
return msg
def process_fds():
path = FilePath(b"/dev/fd")
if not path.exists():
raise SkipTest("/dev/fd is not available.")
return set([child.basename() for child in path.children()])
def create_proxy_to(logger, ip, port):
"""
:see: ``HostNetwork.create_proxy_to``
"""
action = CREATE_PROXY_TO(logger=logger, target_ip=ip, target_port=port)
with action:
encoded_ip = unicode(ip).encode("ascii")
encoded_port = unicode(port).encode("ascii")
# The first goal is to configure "Destination NAT" (DNAT). We're just
# going to rewrite the destination address of traffic arriving on the
# specified port so it looks like it is destined for the specified ip
# instead of destined for "us". This gets the packets delivered to the
# right destination.
iptables(
logger,
[
# All NAT stuff happens in the netfilter NAT table.
b"--table",
b"nat",
# Destination NAT has to happen "pre"-routing so that the normal
# routing rules on the machine will use the re-written destination
# address and get the packet to that new destination. Accomplish
# this by appending the rule to the PREROUTING chain.
b"--append",
b"PREROUTING",
# Only re-route traffic with a destination port matching the one we
# were told to manipulate. It is also necessary to specify TCP (or
# UDP) here since that is the layer of the network stack that
# defines ports.
b"--protocol",
b"tcp",
b"--destination-port",
encoded_port,
# And only re-route traffic directed at this host. Traffic
# originating on this host directed at some random other host that
# happens to be on the same port should be left alone.
b"--match",
b"addrtype",
b"--dst-type",
b"LOCAL",
# Tag it as a flocker-created rule so we can recognize it later.
b"--match",
b"comment",
b"--comment",
FLOCKER_PROXY_COMMENT_MARKER,
# If the filter matched, jump to the DNAT chain to handle doing the
# actual packet mangling. DNAT is a built-in chain that already
# knows how to do this. Pass an argument to the DNAT chain so it
# knows how to mangle the packet - rewrite the destination IP of
# the address to the target we were told to use.
b"--jump",
b"DNAT",
b"--to-destination",
encoded_ip,
])
# Bonus round! Having performed DNAT (changing the destination) during
# prerouting we are now prepared to send the packet on somewhere else.
# On its way out of this system it is also necessary to further
# modify and then track that packet. We want it to look like it
# comes from us (the downstream client will be *very* confused if
# the node we're passing the packet on to replies *directly* to them;
# and by confused I mean it will be totally broken, of course) so we
# also need to "masquerade" in the postrouting chain. This changes
# the source address (ip and port) of the packet to the address of
# the external interface the packet is exiting upon. Doing SNAT here
# would be a little bit more efficient because the kernel could avoid
# looking up the external interface's address for every single packet.
# But it requires this code to know that address and it requires that
# if it ever changes the rule gets updated and it may require some
# steps to do port allocation (not sure what they are yet). So we'll
# just masquerade for now.
iptables(
logger,
[
# All NAT stuff happens in the netfilter NAT table.
b"--table",
b"nat",
# As described above, this transformation happens after routing
# decisions have been made and the packet is on its way out of the
# system. Therefore, append the rule to the POSTROUTING chain.
b"--append",
b"POSTROUTING",
# We'll stick to matching the same kinds of packets we matched in
# the earlier stage. We might want to change the factoring of this
# code to avoid the duplication - particularly in case we want to
# change the specifics of the filter.
#
# This omits the LOCAL addrtype check, though, because at this
# point the packet is definitely leaving this host.
b"--protocol",
b"tcp",
b"--destination-port",
encoded_port,
# Do the masquerading.
b"--jump",
b"MASQUERADE",
])
# Secret level!! Traffic that originates *on* the host bypasses the
# PREROUTING chain. Instead, it passes through the OUTPUT chain. If
# we want connections from localhost to the forwarded port to be
# affected then we need a rule in the OUTPUT chain to do the same kind
# of DNAT that we did in the PREROUTING chain.
iptables(
logger,
[
# All NAT stuff happens in the netfilter NAT table.
b"--table",
b"nat",
# As mentioned, this rule is for the OUTPUT chain.
b"--append",
b"OUTPUT",
# Matching the exact same kinds of packets as the PREROUTING rule
# matches.
b"--protocol",
b"tcp",
b"--destination-port",
encoded_port,
b"--match",
b"addrtype",
b"--dst-type",
b"LOCAL",
# Do the same DNAT as we did in the rule for the PREROUTING chain.
b"--jump",
b"DNAT",
b"--to-destination",
encoded_ip,
])
iptables(logger, [
b"--table",
b"filter",
b"--insert",
b"FORWARD",
b"--destination",
encoded_ip,
b"--protocol",
b"tcp",
b"--destination-port",
encoded_port,
b"--jump",
b"ACCEPT",
])
# The network stack only considers forwarding traffic when certain
# system configuration is in place.
#
# https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt
# will explain the meaning of these in (very slightly) more detail.
conf = FilePath(b"/proc/sys/net/ipv4/conf")
descendant = conf.descendant([b"default", b"forwarding"])
with descendant.open("wb") as forwarding:
forwarding.write(b"1")
# In order to have the OUTPUT chain DNAT rule affect routing decisions,
# we also need to tell the system to make routing decisions about
# traffic from or to localhost.
for path in conf.children():
with path.child(b"route_localnet").open("wb") as route_localnet:
route_localnet.write(b"1")
return Proxy(ip=ip, port=port)
class MapUpdater(object):
def __init__(self, mapsPath, fetchURL, deleteIfNotPresent, tfLevelSounds):
assert isinstance(mapsPath, str) and len(mapsPath)
assert isinstance(fetchURL, str) and len(fetchURL)
self.mapsPath = FilePath(mapsPath)
self.downloadTempPath = self.mapsPath.child('mapupdater')
self.fetchURL = URLPath.fromString(fetchURL)
self.deleteIfNotPresent = deleteIfNotPresent
self.tfLevelSounds = tfLevelSounds
self.semaphore = DeferredSemaphore(1)
self.downloadSemaphore = DeferredSemaphore(4)
for fp in self.downloadTempPath.globChildren('*.bsp.bz2'):
fp.remove()
def checkMaps(self, *a, **kw):
"""
Wrap self._checkMaps to prevent running multiple checks at once.
"""
return self.semaphore.run(self._checkMaps, *a, **kw)
def _checkMaps(self, forceDownloadMaps=None):
def _cb(remoteMaps):
if forceDownloadMaps:
remoteMaps = list(set(remoteMaps + forceDownloadMaps))
remoteMapsLower = [f.lower() for f in remoteMaps]
ourMaps = filter(lambda p: not p.isdir() and p.path.endswith('.bsp'),
self.mapsPath.children())
ourMapFilenames = [p.basename().lower() + '.bz2' for p in ourMaps]
missing = []
for f in remoteMaps:
if f.lower() not in ourMapFilenames:
missing.append(f)
delete = []
for p in ourMaps:
filename = p.basename().lower() + '.bz2'
if filename not in remoteMapsLower:
delete.append(p)
if self.deleteIfNotPresent and delete:
for fp in delete:
fp.remove()
print 'Deleted {} map(s) not present at remote server:'.format(len(delete))
print ', '.join([x.basename() for x in delete])
if missing:
print 'Fetching {} map(s)'.format(len(missing))
def _allFinished(ignored):
self.mapsPath.child('tempus_map_updater_run_once').touch()
if self.tfLevelSounds:
self.addLevelSounds(ourMaps)
print 'Now up-to-date.'
ds = []
for filename in missing:
ds.append(self.fetchMap(filename))
return gatherResults(ds).addCallback(_allFinished)
elif self.tfLevelSounds:
self.addLevelSounds(ourMaps)
return self.getMapList().addCallback(_cb)
def fetchMap(self, *a, **kw):
return self.downloadSemaphore.run(self._fetchMap, *a, **kw)
def _fetchMap(self, filename):
downloadTempPath = self.downloadTempPath
if not downloadTempPath.exists():
downloadTempPath.makedirs()
def _cb(response, fn):
tp = downloadTempPath.child(fn)
fd = tp.open('wb')
def _extracted(ignored):
extractedPath = tp.sibling(tp.basename().replace('.bz2', ''))
extractedPath.moveTo(
self.mapsPath.child(tp.basename().replace('.bz2', '')))
try:
tp.remove()
# File already gone
except OSError:
pass
print 'Finished downloading {}'.format(fn)
def _finished(ignored):
fd.close()
d = getProcessOutputAndValue(
'aunpack', (tp.path, '-X', downloadTempPath.path))
d.addErrback(log.err)
d.addCallback(_extracted)
return d
def _eb(failure):
print 'Error downloading {}:'.format(fn)
print failure.getTraceback()
fd.close()
try:
tp.remove()
# File already gone
except OSError:
pass
d = treq.collect(response, fd.write)
d.addCallback(_finished)
d.addErrback(_eb)
return d
d = treq.get(str(self.fetchURL.child(filename)))
return d.addCallback(_cb, filename)
def getMapList(self, forceDownloadMaps):
raise NotImplementedError('Subclasses must override this method.')
def addLevelSounds(self, mapPaths):
content = FilePath(mapupdater.__file__).sibling(
'tf_level_sounds.txt').getContent()
added = []
for p in mapPaths:
mapName = p.basename()[:-4]
p2 = p.sibling('{}_level_sounds.txt'.format(mapName))
if p2.exists() and p2.getContent() == content:
continue
added.append(mapName)
p2.setContent(content)
if added:
print 'Added level sounds for:'
print ', '.join(added)
def create_proxy_to(logger, ip, port):
"""
:see: ``HostNetwork.create_proxy_to``
"""
action = CREATE_PROXY_TO(
logger=logger, target_ip=ip, target_port=port)
with action:
encoded_ip = unicode(ip).encode("ascii")
encoded_port = unicode(port).encode("ascii")
# The first goal is to configure "Destination NAT" (DNAT). We're just
# going to rewrite the destination address of traffic arriving on the
# specified port so it looks like it is destined for the specified ip
# instead of destined for "us". This gets the packets delivered to the
# right destination.
iptables(logger, [
# All NAT stuff happens in the netfilter NAT table.
b"--table", b"nat",
# Destination NAT has to happen "pre"-routing so that the normal
# routing rules on the machine will use the re-written destination
# address and get the packet to that new destination. Accomplish
# this by appending the rule to the PREROUTING chain.
b"--append", b"PREROUTING",
# Only re-route traffic with a destination port matching the one we
# were told to manipulate. It is also necessary to specify TCP (or
# UDP) here since that is the layer of the network stack that
# defines ports.
b"--protocol", b"tcp", b"--destination-port", encoded_port,
# And only re-route traffic directed at this host. Traffic
# originating on this host directed at some random other host that
# happens to be on the same port should be left alone.
b"--match", b"addrtype", b"--dst-type", b"LOCAL",
# Tag it as a flocker-created rule so we can recognize it later.
b"--match", b"comment", b"--comment", FLOCKER_PROXY_COMMENT_MARKER,
# If the filter matched, jump to the DNAT chain to handle doing the
# actual packet mangling. DNAT is a built-in chain that already
# knows how to do this. Pass an argument to the DNAT chain so it
# knows how to mangle the packet - rewrite the destination IP of
# the address to the target we were told to use.
b"--jump", b"DNAT", b"--to-destination", encoded_ip,
])
# Bonus round! Having performed DNAT (changing the destination) during
# prerouting we are now prepared to send the packet on somewhere else.
# On its way out of this system it is also necessary to further
# modify and then track that packet. We want it to look like it
# comes from us (the downstream client will be *very* confused if
# the node we're passing the packet on to replies *directly* to them;
# and by confused I mean it will be totally broken, of course) so we
# also need to "masquerade" in the postrouting chain. This changes
# the source address (ip and port) of the packet to the address of
# the external interface the packet is exiting upon. Doing SNAT here
# would be a little bit more efficient because the kernel could avoid
# looking up the external interface's address for every single packet.
# But it requires this code to know that address and it requires that
# if it ever changes the rule gets updated and it may require some
# steps to do port allocation (not sure what they are yet). So we'll
# just masquerade for now.
iptables(logger, [
# All NAT stuff happens in the netfilter NAT table.
b"--table", b"nat",
# As described above, this transformation happens after routing
# decisions have been made and the packet is on its way out of the
# system. Therefore, append the rule to the POSTROUTING chain.
b"--append", b"POSTROUTING",
# We'll stick to matching the same kinds of packets we matched in
# the earlier stage. We might want to change the factoring of this
# code to avoid the duplication - particularly in case we want to
# change the specifics of the filter.
#
# This omits the LOCAL addrtype check, though, because at this
# point the packet is definitely leaving this host.
b"--protocol", b"tcp", b"--destination-port", encoded_port,
# Do the masquerading.
b"--jump", b"MASQUERADE",
])
# Secret level!! Traffic that originates *on* the host bypasses the
# PREROUTING chain. Instead, it passes through the OUTPUT chain. If
# we want connections from localhost to the forwarded port to be
# affected then we need a rule in the OUTPUT chain to do the same kind
# of DNAT that we did in the PREROUTING chain.
iptables(logger, [
# All NAT stuff happens in the netfilter NAT table.
b"--table", b"nat",
# As mentioned, this rule is for the OUTPUT chain.
b"--append", b"OUTPUT",
# Matching the exact same kinds of packets as the PREROUTING rule
# matches.
b"--protocol", b"tcp",
b"--destination-port", encoded_port,
b"--match", b"addrtype", b"--dst-type", b"LOCAL",
# Do the same DNAT as we did in the rule for the PREROUTING chain.
b"--jump", b"DNAT", b"--to-destination", encoded_ip,
])
iptables(logger, [
b"--table", b"filter",
b"--insert", b"FORWARD",
b"--destination", encoded_ip,
b"--protocol", b"tcp", b"--destination-port", encoded_port,
b"--jump", b"ACCEPT",
])
# The network stack only considers forwarding traffic when certain
# system configuration is in place.
#
# https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt
# will explain the meaning of these in (very slightly) more detail.
conf = FilePath(b"/proc/sys/net/ipv4/conf")
descendant = conf.descendant([b"default", b"forwarding"])
with descendant.open("wb") as forwarding:
forwarding.write(b"1")
# In order to have the OUTPUT chain DNAT rule affect routing decisions,
# we also need to tell the system to make routing decisions about
# traffic from or to localhost.
for path in conf.children():
with path.child(b"route_localnet").open("wb") as route_localnet:
route_localnet.write(b"1")
return Proxy(ip=ip, port=port)
def process_fds():
path = FilePath(b"/proc/self/fd")
return set([child.basename() for child in path.children()])
class Voluminous(object):
lockFactory = DockerLock
def __init__(self, directory):
self._directory = FilePath(directory)
self._output = []
self.lock = self.lockFactory()
self.commitDatabase = JsonCommitDatabase(self._directory)
def output(self, s):
self._output.append(s)
print s
def getOutput(self):
return self._output
def allBranches(self, volume):
volumePath = self._directory.child(volume)
branches = volumePath.child("branches").children()
return [b.basename() for b in branches if b.isdir()]
def listBranches(self):
volume = self.volume()
branches = self.allBranches(volume)
currentBranch = self.getActiveBranch(volume)
self.output("\n".join(
sorted(("*" if b == currentBranch else " ") + " " + b
for b in branches)))
def checkoutBranch(self, branch, create):
"""
"Check out" a branch, restarting containers in process, creating it
from current branch HEAD if requested.
"""
volume = self.volume()
volumePath = self._directory.child(volume)
branchPath = volumePath.child("branches").child(branch)
if create:
if branchPath.exists():
self.output("Cannot create existing branch %s" % (branch, ))
return
else:
try:
HEAD = self._resolveNamedCommitCurrentBranch(
"HEAD", volume)
except IndexError:
self.output(
"You must commit ('dvol commit') before you can "
"branch ('dvol checkout -b')")
return
# Copy metadata
meta = self.commitDatabase.read(volume,
self.getActiveBranch(volume))
self.commitDatabase.write(volume, branch, meta)
# Then copy latest HEAD of branch into new branch data
# directory
volumePath.child("commits").child(HEAD).copyTo(branchPath)
else:
if not branchPath.exists():
self.output("Cannot switch to non-existing branch %s" %
(branch, ))
return
# Got here, so switch to the (maybe new branch)
self.setActiveBranch(volume, branch)
def createBranch(self, volume, branch):
branchDir = self._directory.child(volume).child("branches").child(
branch)
branchDir.makedirs()
self.output("Created branch %s/%s" % (volume, branch))
def createVolume(self, name):
if self._directory.child(name).exists():
self.output("Error: volume %s already exists" % (name, ))
raise VolumeAlreadyExists()
self._directory.child(name).makedirs()
self.setActiveVolume(name)
self.output("Created volume %s" % (name, ))
self.createBranch(name, DEFAULT_BRANCH)
def removeVolume(self, volume):
if not self._directory.child(volume).exists():
raise UsageError("Volume %r does not exist, cannot remove it" %
(volume, ))
containers = self.lock.containers.get_related_containers(volume)
if containers:
raise UsageError("Cannot remove %r while it is in use by '%s'" %
(volume, (",".join(c['Name']
for c in containers))))
if self._userIsSure(
"This will remove all containers using the volume"):
self.output("Deleting volume %r" % (volume, ))
# Remove related containers
self.lock.containers.remove_related_containers(volume)
self._directory.child(volume).remove()
else:
self.output("Aborting.")
def deleteBranch(self, branch):
volume = self.volume()
if branch == self.getActiveBranch(volume):
raise UsageError("Cannot delete active branch, use "
"'dvol checkout' to switch branches first")
if branch not in self.allBranches(volume):
raise UsageError("Branch %r does not exist" % (branch, ))
if self._userIsSure():
self.output("Deleting branch %r" % (branch, ))
volumePath = self._directory.child(volume)
branchPath = volumePath.child("branches").child(branch)
branchPath.remove()
else:
self.output("Aborting.")
def _userIsSure(self, extraMessage=None):
message = "Are you sure? "
if extraMessage:
message += extraMessage
message += " (y/n): "
sys.stdout.write(message)
sys.stdout.flush()
return raw_input().lower() in ("y", "yes")
def setActiveVolume(self, volume):
self._directory.child("current_volume.json").setContent(
json.dumps(dict(current_volume=volume)))
def volume(self):
currentVolume = self._directory.child("current_volume.json")
if currentVolume.exists():
volume = json.loads(currentVolume.getContent())["current_volume"]
else:
raise UsageError("No active volume: use dvol switch to choose one")
if not self._directory.child(volume).exists():
raise UsageError("Active volume %s does not exist: "
"use dvol switch to choose another" % (volume, ))
return volume
def setActiveBranch(self, volume, branch):
self._directory.child(volume).child("current_branch.json").setContent(
json.dumps(dict(current_branch=branch)))
self.lock.acquire(volume)
try:
self.updateRunningPoint(volume)
finally:
self.lock.release(volume)
def getActiveBranch(self, volume):
currentBranch = self._directory.child(
self.volume()).child("current_branch.json")
if currentBranch.exists():
return json.loads(currentBranch.getContent())["current_branch"]
else:
return DEFAULT_BRANCH
def updateRunningPoint(self, volume):
"""
construct a stable (wrt switching branches) path with symlinks
"""
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
stablePath = volumePath.child("running_point")
if stablePath.exists():
stablePath.remove()
branchPath.linkTo(stablePath)
return stablePath.path
def commitVolume(self, message):
volume = self.volume()
commitId = (str(uuid.uuid4()) + str(uuid.uuid4())).replace("-",
"")[:40]
self.output(commitId)
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
commitPath = volumePath.child("commits").child(commitId)
if commitPath.exists():
raise Exception("woah, random uuid collision. try again!")
commitPath.makedirs()
# acquire lock (read: stop containers) to ensure consistent snapshot
# with file-copy based backend
# XXX tests for acquire/release
self.lock.acquire(volume)
try:
branchPath.copyTo(commitPath)
finally:
self.lock.release(volume)
self._recordCommit(volume, branchName, commitId, message)
def _recordCommit(self, volume, branch, commitId, message):
commitData = self.commitDatabase.read(volume, branch)
commitData.append(dict(id=commitId, message=message))
self.commitDatabase.write(volume, branch, commitData)
def exists(self, volume):
volumePath = self._directory.child(volume)
return volumePath.exists()
def listVolumes(self):
table = get_table()
table.set_cols_align(["l", "l", "l"])
dc = self.lock.containers # XXX ugly
volumes = [v for v in self._directory.children() if v.isdir()]
activeVolume = None
if volumes:
try:
activeVolume = self.volume()
except UsageError:
# don't refuse to list volumes just because none of them are active
pass
rows = [["", "", ""]] + [[" VOLUME", "BRANCH", "CONTAINERS"]] + [[
("*" if v.basename() == activeVolume else " ") + " " +
v.basename(),
self.getActiveBranch(v.basename()), ",".join(
c['Name'] for c in dc.get_related_containers(v.basename()))
] for v in volumes]
table.add_rows(rows)
self.output(table.draw())
def listCommits(self, branch=None):
if branch is None:
branch = self.getActiveBranch(self.volume())
volume = self.volume()
aggregate = []
for commit in reversed(self.commitDatabase.read(volume, branch)):
# TODO fill in author/date
aggregate.append("commit %(id)s\n"
"Author: Who knows <mystery@person>\n"
"Date: Whenever\n"
"\n"
" %(message)s\n" % commit)
self.output("\n".join(aggregate))
def _resolveNamedCommitCurrentBranch(self, commit, volume):
branch = self.getActiveBranch(volume)
remainder = commit[len("HEAD"):]
if remainder == "^" * len(remainder):
offset = len(remainder)
else:
raise UsageError("Malformed commit identifier %r" % (commit, ))
commits = self.commitDatabase.read(volume, branch)
# commits are appended to, so the last one is the latest
return commits[-1 - offset]["id"]
def _destroyNewerCommits(self, commit, volume):
# TODO in the future, we'll care more about the following being an
# atomic operation
branch = self.getActiveBranch(volume)
commits = self.commitDatabase.read(volume, branch)
commitIndex = [c["id"] for c in commits].index(commit) + 1
remainingCommits = commits[:commitIndex]
destroyCommits = commits[commitIndex:]
# look in all branches for commit references before removing them
totalCommits = set()
for otherBranch in self.allBranches(volume):
if otherBranch == branch:
# skip this branch, otherwise we'll never destroy any commits
continue
commits = self.commitDatabase.read(volume, branch)
totalCommits.update(commit["id"] for commit in commits)
for commit in destroyCommits:
commitId = commit["id"]
if commitId in totalCommits:
# skip destroying this commit; it is still actively referred to
# in another branch
continue
volumePath = self._directory.child(volume)
commitPath = volumePath.child("commits").child(commitId)
commitPath.remove()
self.commitDatabase.write(volume, branch, remainingCommits)
def resetVolume(self, commit):
"""
Forcefully roll back the current working copy to this commit,
destroying any later commits.
"""
volume = self.volume()
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
if commit.startswith("HEAD"):
try:
commit = self._resolveNamedCommitCurrentBranch(commit, volume)
except IndexError:
self.output("Referenced commit does not exist; check dvol log")
return
commitPath = volumePath.child("commits").child(commit)
if not commitPath.exists():
raise NoSuchCommit("commit '%s' does not exist" % (commit, ))
self.lock.acquire(volume)
try:
branchPath.remove()
commitPath.copyTo(branchPath)
self._destroyNewerCommits(commit, volume)
finally:
self.lock.release(volume)
def process_fds():
path = FilePath(b"/proc/self/fd")
if not path.exists():
raise SkipTest("/proc is not available.")
return set([child.basename() for child in path.children()])
class NewsBuilderTests(TestCase, StructureAssertingMixin):
"""
Tests for L{NewsBuilder}.
"""
skip = svnSkip
def setUp(self):
"""
Create a fake project and stuff some basic structure and content into
it.
"""
self.builder = NewsBuilder()
self.project = FilePath(self.mktemp())
self.project.createDirectory()
self.existingText = 'Here is stuff which was present previously.\n'
createStructure(
self.project, {
'NEWS': self.existingText,
'5.feature': 'We now support the web.\n',
'12.feature': 'The widget is more robust.\n',
'15.feature': (
'A very long feature which takes many words to '
'describe with any accuracy was introduced so that '
'the line wrapping behavior of the news generating '
'code could be verified.\n'),
'16.feature': (
'A simpler feature\ndescribed on multiple lines\n'
'was added.\n'),
'23.bugfix': 'Broken stuff was fixed.\n',
'25.removal': 'Stupid stuff was deprecated.\n',
'30.misc': '',
'35.misc': '',
'40.doc': 'foo.bar.Baz.quux',
'41.doc': 'writing Foo servers'})
def test_today(self):
"""
L{NewsBuilder._today} returns today's date in YYYY-MM-DD form.
"""
self.assertEqual(
self.builder._today(), date.today().strftime('%Y-%m-%d'))
def test_findFeatures(self):
"""
When called with L{NewsBuilder._FEATURE}, L{NewsBuilder._findChanges}
returns a list of bugfix ticket numbers and descriptions as a list of
two-tuples.
"""
features = self.builder._findChanges(
self.project, self.builder._FEATURE)
self.assertEqual(
features,
[(5, "We now support the web."),
(12, "The widget is more robust."),
(15,
"A very long feature which takes many words to describe with "
"any accuracy was introduced so that the line wrapping behavior "
"of the news generating code could be verified."),
(16, "A simpler feature described on multiple lines was added.")])
def test_findBugfixes(self):
"""
When called with L{NewsBuilder._BUGFIX}, L{NewsBuilder._findChanges}
returns a list of bugfix ticket numbers and descriptions as a list of
two-tuples.
"""
bugfixes = self.builder._findChanges(
self.project, self.builder._BUGFIX)
self.assertEqual(
bugfixes,
[(23, 'Broken stuff was fixed.')])
def test_findRemovals(self):
"""
When called with L{NewsBuilder._REMOVAL}, L{NewsBuilder._findChanges}
returns a list of removal/deprecation ticket numbers and descriptions
as a list of two-tuples.
"""
removals = self.builder._findChanges(
self.project, self.builder._REMOVAL)
self.assertEqual(
removals,
[(25, 'Stupid stuff was deprecated.')])
def test_findDocumentation(self):
"""
When called with L{NewsBuilder._DOC}, L{NewsBuilder._findChanges}
returns a list of documentation ticket numbers and descriptions as a
list of two-tuples.
"""
doc = self.builder._findChanges(
self.project, self.builder._DOC)
self.assertEqual(
doc,
[(40, 'foo.bar.Baz.quux'),
(41, 'writing Foo servers')])
def test_findMiscellaneous(self):
"""
When called with L{NewsBuilder._MISC}, L{NewsBuilder._findChanges}
returns a list of removal/deprecation ticket numbers and descriptions
as a list of two-tuples.
"""
misc = self.builder._findChanges(
self.project, self.builder._MISC)
self.assertEqual(
misc,
[(30, ''),
(35, '')])
def test_writeHeader(self):
"""
L{NewsBuilder._writeHeader} accepts a file-like object opened for
writing and a header string and writes out a news file header to it.
"""
output = StringIO()
self.builder._writeHeader(output, "Super Awesometastic 32.16")
self.assertEqual(
output.getvalue(),
"Super Awesometastic 32.16\n"
"=========================\n"
"\n")
def test_writeSection(self):
"""
L{NewsBuilder._writeSection} accepts a file-like object opened for
writing, a section name, and a list of ticket information (as returned
by L{NewsBuilder._findChanges}) and writes out a section header and all
of the given ticket information.
"""
output = StringIO()
self.builder._writeSection(
output, "Features",
[(3, "Great stuff."),
(17, "Very long line which goes on and on and on, seemingly "
"without end until suddenly without warning it does end.")])
self.assertEqual(
output.getvalue(),
"Features\n"
"--------\n"
" - Great stuff. (#3)\n"
" - Very long line which goes on and on and on, seemingly "
"without end\n"
" until suddenly without warning it does end. (#17)\n"
"\n")
def test_writeMisc(self):
"""
L{NewsBuilder._writeMisc} accepts a file-like object opened for
writing, a section name, and a list of ticket information (as returned
by L{NewsBuilder._findChanges} and writes out a section header and all
of the ticket numbers, but excludes any descriptions.
"""
output = StringIO()
self.builder._writeMisc(
output, "Other",
[(x, "") for x in range(2, 50, 3)])
self.assertEqual(
output.getvalue(),
"Other\n"
"-----\n"
" - #2, #5, #8, #11, #14, #17, #20, #23, #26, #29, #32, #35, "
"#38, #41,\n"
" #44, #47\n"
"\n")
def test_build(self):
"""
L{NewsBuilder.build} updates a NEWS file with new features based on the
I{<ticket>.feature} files found in the directory specified.
"""
self.builder.build(
self.project, self.project.child('NEWS'),
"Super Awesometastic 32.16")
results = self.project.child('NEWS').getContent()
self.assertEqual(
results,
'Super Awesometastic 32.16\n'
'=========================\n'
'\n'
'Features\n'
'--------\n'
' - We now support the web. (#5)\n'
' - The widget is more robust. (#12)\n'
' - A very long feature which takes many words to describe '
'with any\n'
' accuracy was introduced so that the line wrapping behavior '
'of the\n'
' news generating code could be verified. (#15)\n'
' - A simpler feature described on multiple lines was '
'added. (#16)\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Improved Documentation\n'
'----------------------\n'
' - foo.bar.Baz.quux (#40)\n'
' - writing Foo servers (#41)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n'
'Other\n'
'-----\n'
' - #30, #35\n'
'\n\n' + self.existingText)
def test_emptyProjectCalledOut(self):
"""
If no changes exist for a project, I{NEWS} gains a new section for
that project that includes some helpful text about how there were no
interesting changes.
"""
project = FilePath(self.mktemp()).child("twisted")
project.makedirs()
createStructure(project, {'NEWS': self.existingText})
self.builder.build(
project, project.child('NEWS'),
"Super Awesometastic 32.16")
results = project.child('NEWS').getContent()
self.assertEqual(
results,
'Super Awesometastic 32.16\n'
'=========================\n'
'\n' +
self.builder._NO_CHANGES +
'\n\n' + self.existingText)
def test_preserveTicketHint(self):
"""
If a I{NEWS} file begins with the two magic lines which point readers
at the issue tracker, those lines are kept at the top of the new file.
"""
news = self.project.child('NEWS')
news.setContent(
'Ticket numbers in this file can be looked up by visiting\n'
'http://twistedmatrix.com/trac/ticket/<number>\n'
'\n'
'Blah blah other stuff.\n')
self.builder.build(self.project, news, "Super Awesometastic 32.16")
self.assertEqual(
news.getContent(),
'Ticket numbers in this file can be looked up by visiting\n'
'http://twistedmatrix.com/trac/ticket/<number>\n'
'\n'
'Super Awesometastic 32.16\n'
'=========================\n'
'\n'
'Features\n'
'--------\n'
' - We now support the web. (#5)\n'
' - The widget is more robust. (#12)\n'
' - A very long feature which takes many words to describe '
'with any\n'
' accuracy was introduced so that the line wrapping behavior '
'of the\n'
' news generating code could be verified. (#15)\n'
' - A simpler feature described on multiple lines was '
'added. (#16)\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Improved Documentation\n'
'----------------------\n'
' - foo.bar.Baz.quux (#40)\n'
' - writing Foo servers (#41)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n'
'Other\n'
'-----\n'
' - #30, #35\n'
'\n\n'
'Blah blah other stuff.\n')
def test_emptySectionsOmitted(self):
"""
If there are no changes of a particular type (feature, bugfix, etc), no
section for that type is written by L{NewsBuilder.build}.
"""
for ticket in self.project.children():
if ticket.splitext()[1] in ('.feature', '.misc', '.doc'):
ticket.remove()
self.builder.build(
self.project, self.project.child('NEWS'),
'Some Thing 1.2')
self.assertEqual(
self.project.child('NEWS').getContent(),
'Some Thing 1.2\n'
'==============\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n\n'
'Here is stuff which was present previously.\n')
def test_duplicatesMerged(self):
"""
If two change files have the same contents, they are merged in the
generated news entry.
"""
def feature(s):
return self.project.child(s + '.feature')
feature('5').copyTo(feature('15'))
feature('5').copyTo(feature('16'))
self.builder.build(
self.project, self.project.child('NEWS'),
'Project Name 5.0')
self.assertEqual(
self.project.child('NEWS').getContent(),
'Project Name 5.0\n'
'================\n'
'\n'
'Features\n'
'--------\n'
' - We now support the web. (#5, #15, #16)\n'
' - The widget is more robust. (#12)\n'
'\n'
'Bugfixes\n'
'--------\n'
' - Broken stuff was fixed. (#23)\n'
'\n'
'Improved Documentation\n'
'----------------------\n'
' - foo.bar.Baz.quux (#40)\n'
' - writing Foo servers (#41)\n'
'\n'
'Deprecations and Removals\n'
'-------------------------\n'
' - Stupid stuff was deprecated. (#25)\n'
'\n'
'Other\n'
'-----\n'
' - #30, #35\n'
'\n\n'
'Here is stuff which was present previously.\n')
class FileUploadResource(Resource):
"""
Twisted Web resource that handles file uploads over `HTTP/POST` requests.
"""
log = make_logger()
def __init__(self,
upload_directory,
temp_directory,
form_fields,
upload_session,
options=None):
"""
:param upload_directory: The target directory where uploaded files will be stored.
:type upload_directory: str
:param temp_directory: A temporary directory where chunks of a file being uploaded are stored.
:type temp_directory: str
:param form_fields: Names of HTML form fields used for uploading.
:type form_fields: dict
:param upload_session: An instance of `ApplicationSession` used for publishing progress events.
:type upload_session: obj
:param options: Options for file upload.
:type options: dict or None
"""
Resource.__init__(self)
self._uploadRoot = FilePath(upload_directory)
self._tempDirRoot = FilePath(temp_directory)
self._form_fields = form_fields
self._fileupload_session = upload_session
self._options = options or {}
self._max_file_size = self._options.get('max_file_size', 10 * 1024 * 1024)
self._fileTypes = self._options.get('file_types', None)
self._file_permissions = self._options.get('file_permissions', None)
# track uploaded files / chunks
self._uploads = {}
self.log.info('Upload Resource started.')
# scan the temp dir for uploaded chunks and fill the _uploads dict with it
# so existing uploads can be resumed
# all other remains will be purged
for fileTempDir in self._tempDirRoot.children():
fileTempName = fileTempDir.basename()
if fileTempDir.isdir():
self._uploads[fileTempName] = {'chunk_list': [], 'origin': 'startup'}
for chunk in fileTempDir.listdir():
if chunk[:6] == 'chunk_':
self._uploads[fileTempName]['chunk_list'].append(int(chunk[6:]))
else:
fileTempDir.child(chunk).remove()
# if no chunks detected then remove remains completely
if len(self._uploads[fileTempName]['chunk_list']) == 0:
fileTempDir.remove()
self._uploads.pop(fileTempName, None)
else: # fileTempDir is a file remaining from a single chunk upload
fileTempDir.remove()
self.log.debug("Scanned pending uploads: {uploads}", uploads=self._uploads)
def render_POST(self, request):
headers = {x.decode('iso-8859-1'): y.decode('iso-8859-1')
for x, y in request.getAllHeaders().items()}
origin = headers['host']
postFields = cgi.FieldStorage(
fp=request.content,
headers=headers,
environ={"REQUEST_METHOD": "POST"})
f = self._form_fields
filename = postFields[f['file_name']].value
totalSize = int(postFields[f['total_size']].value)
totalChunks = int(postFields[f['total_chunks']].value)
chunkSize = int(postFields[f['chunk_size']].value)
chunkNumber = int(postFields[f['chunk_number']].value)
fileContent = postFields[f['content']].value
if 'chunk_extra' in f and f['chunk_extra'] in postFields:
chunk_extra = json.loads(postFields[f['chunk_extra']].value)
else:
chunk_extra = {}
if 'finish_extra' in f and f['finish_extra'] in postFields:
finish_extra = json.loads(postFields[f['finish_extra']].value)
else:
finish_extra = {}
fileId = filename
# # prepare user specific upload areas
# # NOT YET IMPLEMENTED
# #
# if 'auth_id' in f and f['auth_id'] in postFields:
# auth_id = postFields[f['auth_id']].value
# mydir = os.path.join(self._uploadRoot, auth_id)
# my_temp_dir = os.path.join(self._tempDirRoot, auth_id)
#
# # check if auth_id is a valid directory_name
# #
# if auth_id != auth_id.encode('ascii', 'ignore'):
# msg = "The requestor auth_id must be an ascii string."
# if self._debug:
# log.msg(msg)
# # 415 Unsupported Media Type
# request.setResponseCode(415, msg)
# return msg
# else:
# auth_id = 'anonymous'
# create user specific folder
# mydir = self._uploadRoot
# my_temp_dir = self._tempDirRoot
# if not os.path.exists(mydir):
# os.makedirs(mydir)
# if not os.path.exists(my_temp_dir):
# os.makedirs(my_temp_dir)
# prepare the on_progress publisher
if 'on_progress' in f and f['on_progress'] in postFields and self._fileupload_session != {}:
topic = postFields[f['on_progress']].value
if 'session' in f and f['session'] in postFields:
session = int(postFields[f['session']].value)
publish_options = PublishOptions(eligible=[session])
else:
publish_options = None
def fileupload_publish(payload):
self._fileupload_session.publish(topic, payload, options=publish_options)
else:
def fileupload_publish(payload):
pass
# Register upload right at the start to avoid overlapping upload conflicts
#
if fileId not in self._uploads:
self._uploads[fileId] = {'chunk_list': [], 'origin': origin}
chunk_is_first = True
self.log.debug('Started upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}',
file_name=fileId, total_size=totalSize, total_chunks=totalChunks, chunk_size=chunkSize, chunk_number=chunkNumber)
else:
chunk_is_first = False
# If the chunks are read at startup of crossbar any client may claim and resume the pending upload !
#
upl = self._uploads[fileId]
if upl['origin'] == 'startup':
self.log.debug('Will try to resume upload of file: file_name={file_name}, total_size={total_size}, total_chunks={total_chunks}, chunk_size={chunk_size}, chunk_number={chunk_number}',
file_name=fileId, total_size=totalSize, total_chunks=totalChunks, chunk_size=chunkSize, chunk_number=chunkNumber)
upl['origin'] = origin
else:
# check if another session is uploading this file already
#
if upl['origin'] != origin:
msg = "File being uploaded is already uploaded in a different session."
self.log.debug(msg)
# 409 Conflict
request.setResponseCode(409, msg.encode('utf8'))
return msg.encode('utf8')
else:
# check if the chunk is being uploaded in this very session already
# this should never happen !
if chunkNumber in upl['chunk_list']:
msg = "Chunk beeing uploaded is already uploading."
self.log.debug(msg)
# Don't throw a conflict. This may be a wanted behaviour.
# Even if an upload would be resumable, you don't have to resume.
# 409 Conflict
# request.setResponseCode(409, msg.encode('utf8'))
# return msg.encode('utf8')
# check file size
#
if totalSize > self._max_file_size:
msg = "Size {} of file to be uploaded exceeds maximum {}".format(totalSize, self._max_file_size)
self.log.debug(msg)
# 413 Request Entity Too Large
request.setResponseCode(413, msg.encode('utf8'))
return msg.encode('utf8')
# check file extensions
#
extension = os.path.splitext(filename)[1]
if self._fileTypes and extension not in self._fileTypes:
msg = "Type '{}' of file to be uploaded is in allowed types {}".format(extension, self._fileTypes)
self.log.debug(msg)
# 415 Unsupported Media Type
request.setResponseCode(415, msg.encode('utf8'))
return msg.encode('utf8')
# TODO: check mime type
#
fileTempDir = self._tempDirRoot.child(fileId)
chunkName = fileTempDir.child('chunk_' + str(chunkNumber))
_chunkName = fileTempDir.child('#kfhf3kz412uru578e38viokbjhfvz4w__' + 'chunk_' + str(chunkNumber))
def mergeFile():
# every chunk has to check if it is the last chunk written, except in a single chunk scenario
if totalChunks > 1 and len(self._uploads[fileId]['chunk_list']) >= totalChunks:
# last chunk
self.log.debug('Finished file upload after chunk {chunk_number} with chunk_list {chunk_list}', chunk_number=chunkNumber, chunk_list=self._uploads)
# Merge all files into one file and remove the temp files
# TODO: How to avoid the extra file IO ?
finalFileName = self._uploadRoot.child(fileId)
_finalFileName = fileTempDir.child('#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName.path, 'wb') as _finalFile:
for cn in range(1, totalChunks + 1):
with open(fileTempDir.child('chunk_' + str(cn)).path, 'rb') as ff:
_finalFile.write(ff.read())
_finalFileName.moveTo(finalFileName)
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
finalFileName.chmod(perm)
except Exception as e:
msg = "file upload resource - could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
self._uploads.pop(fileId, None)
request.setResponseCode(500, msg.encode('utf8'))
return msg.encode('utf8')
else:
self.log.debug("Changed permissions on {file_name} to {permissions}", file_name=finalFileName, permissions=self._file_permissions)
# remove the file temp folder
fileTempDir.remove()
self._uploads.pop(fileId, None)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.,
"finish_extra": finish_extra,
"chunk_extra": chunk_extra
})
if chunk_is_first:
# first chunk of file
# publish file upload start
#
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": totalSize,
"status": "started",
"progress": 0.,
"chunk_extra": chunk_extra
})
if totalChunks == 1:
# only one chunk overall -> write file directly
finalFileName = self._uploadRoot.child(fileId)
_finalFileName = self._tempDirRoot.child('#kfhf3kz412uru578e38viokbjhfvz4w__' + fileId)
with open(_finalFileName.path, 'wb') as _finalFile:
_finalFile.write(fileContent)
if self._file_permissions:
perm = int(self._file_permissions, 8)
try:
_finalFileName.chmod(perm)
except Exception as e:
# finalFileName.remove()
msg = "Could not change file permissions of uploaded file"
self.log.debug(msg)
self.log.debug(e)
request.setResponseCode(500, msg.encode('utf8'))
return msg.encode('utf8')
else:
self.log.debug("Changed permissions on {file_name} to {permissions}", file_name=finalFileName, permissions=self._file_permissions)
_finalFileName.moveTo(finalFileName)
if chunkNumber not in self._uploads[fileId]['chunk_list']:
self._uploads[fileId]['chunk_list'].append(chunkNumber)
self._uploads.pop(fileId, None)
# publish file upload progress to file_progress_URI
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": 0,
"status": "finished",
"progress": 1.,
"finish_extra": finish_extra,
"chunk_extra": chunk_extra
})
else:
# first of more chunks
# fileTempDir.remove() # any potential conflict should have been resolved above. This should not be necessary!
if not os.path.isdir(fileTempDir.path):
fileTempDir.makedirs()
with open(_chunkName.path, 'wb') as chunk:
chunk.write(fileContent)
_chunkName.moveTo(chunkName) # atomic file system operation
self.log.debug('chunk_' + str(chunkNumber) + ' written and moved to ' + chunkName.path)
# publish file upload progress
#
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": totalSize - chunkSize,
"status": "progress",
"progress": round(float(chunkSize) / float(totalSize), 3),
"chunk_extra": chunk_extra
})
if chunkNumber not in self._uploads[fileId]['chunk_list']:
self._uploads[fileId]['chunk_list'].append(chunkNumber)
mergeFile()
# clean the temp dir once per file upload
self._remove_stale_uploads()
else:
# intermediate chunk
if not os.path.isdir(fileTempDir.path):
fileTempDir.makedirs()
with open(_chunkName.path, 'wb') as chunk:
chunk.write(fileContent)
_chunkName.moveTo(chunkName)
self.log.debug('chunk_' + str(chunkNumber) + ' written and moved to ' + chunkName.path)
if chunkNumber not in self._uploads[fileId]['chunk_list']:
self._uploads[fileId]['chunk_list'].append(chunkNumber)
received = sum(fileTempDir.child(f).getsize() for f in fileTempDir.listdir())
fileupload_publish({
"id": fileId,
"chunk": chunkNumber,
"name": filename,
"total": totalSize,
"remaining": totalSize - received,
"status": "progress",
"progress": round(float(received) / float(totalSize), 3),
"chunk_extra": chunk_extra
})
mergeFile()
# no errors encountered -> respond success
request.setResponseCode(200)
return b''
def _remove_stale_uploads(self):
"""
This only works if there is a temp folder exclusive for crossbar file uploads
if the system temp folder is used then crossbar creates a "crossbar-uploads" there and
uses that as the temp folder for uploads
If you don't clean up regularly an attacker could fill up the OS file system
"""
for fileTempDir in self._tempDirRoot.children():
self.log.debug('REMOVE STALE UPLOADS ' + str(fileTempDir.basename()))
if fileTempDir.isdir() and (fileTempDir.basename()) not in self._uploads:
fileTempDir.remove()
def render_GET(self, request):
"""
This method can be used to check whether a chunk has been uploaded already.
It returns with HTTP status code `200` if yes and `404` if not.
The request needs to contain the file identifier and the chunk number to check for.
"""
for param in ['file_name', 'chunk_number']:
if not self._form_fields[param].encode('iso-8859-1') in request.args:
msg = "file upload resource - missing request query parameter '{}', configured from '{}'".format(self._form_fields[param], param)
self.log.debug(msg)
# 400 Bad Request
request.setResponseCode(400, msg.encode('utf8'))
return msg.encode('utf8')
file_name = request.args[self._form_fields['file_name'].encode('iso-8859-1')][0].decode('utf8')
chunk_number = int(request.args[self._form_fields['chunk_number'].encode('iso-8859-1')][0].decode('utf8'))
# a complete upload will be repeated an incomplete upload will be resumed
if file_name in self._uploads and chunk_number in self._uploads[file_name]['chunk_list']:
self.log.debug("Skipping chunk upload {file_name} of chunk {chunk_number}", file_name=file_name, chunk_number=chunk_number)
msg = b"chunk of file already uploaded"
request.setResponseCode(200, msg)
return msg
else:
msg = b"chunk of file not yet uploaded"
request.setResponseCode(404, msg)
return msg
class Voluminous(object):
lockFactory = DockerLock
def __init__(self, directory):
self._directory = FilePath(directory)
self._output = []
self.lock = self.lockFactory()
self.commitDatabase = JsonCommitDatabase(self._directory)
def output(self, s):
self._output.append(s)
print s
def getOutput(self):
return self._output
def allBranches(self, volume):
volumePath = self._directory.child(volume)
branches = volumePath.child("branches").children()
return [b.basename() for b in branches if b.isdir()]
def listBranches(self):
volume = self.volume()
branches = self.allBranches(volume)
currentBranch = self.getActiveBranch(volume)
self.output("\n".join(sorted(
("*" if b == currentBranch else " ")
+ " " + b for b in branches)))
def checkoutBranch(self, branch, create):
"""
"Check out" a branch, restarting containers in process, creating it
from current branch HEAD if requested.
"""
volume = self.volume()
volumePath = self._directory.child(volume)
branchPath = volumePath.child("branches").child(branch)
if create:
if branchPath.exists():
self.output("Cannot create existing branch %s" % (branch,))
return
else:
try:
HEAD = self._resolveNamedCommitCurrentBranch("HEAD", volume)
except IndexError:
self.output("You must commit ('dvol commit') before you can "
"branch ('dvol checkout -b')")
return
# Copy metadata
meta = self.commitDatabase.read(volume,
self.getActiveBranch(volume))
self.commitDatabase.write(volume, branch, meta)
# Then copy latest HEAD of branch into new branch data
# directory
volumePath.child("commits").child(HEAD).copyTo(branchPath)
else:
if not branchPath.exists():
self.output("Cannot switch to non-existing branch %s" % (branch,))
return
# Got here, so switch to the (maybe new branch)
self.setActiveBranch(volume, branch)
def createBranch(self, volume, branch):
branchDir = self._directory.child(volume).child("branches").child(branch)
branchDir.makedirs()
self.output("Created branch %s/%s" % (volume, branch))
def createVolume(self, name):
if self._directory.child(name).exists():
self.output("Error: volume %s already exists" % (name,))
raise VolumeAlreadyExists()
self._directory.child(name).makedirs()
self.setActiveVolume(name)
self.output("Created volume %s" % (name,))
self.createBranch(name, DEFAULT_BRANCH)
def removeVolume(self, volume):
if not self._directory.child(volume).exists():
raise UsageError("Volume %r does not exist, cannot remove it" %
(volume,))
containers = self.lock.containers.get_related_containers(volume)
if containers:
raise UsageError("Cannot remove %r while it is in use by '%s'" %
(volume, (",".join(c['Name'] for c in containers))))
if self._userIsSure():
self.output("Deleting volume %r" % (volume,))
self._directory.child(volume).remove()
else:
self.output("Aborting.")
def deleteBranch(self, branch):
volume = self.volume()
if branch == self.getActiveBranch(volume):
raise UsageError("Cannot delete active branch, use "
"'dvol checkout' to switch branches first")
if branch not in self.allBranches(volume):
raise UsageError("Branch %r does not exist" % (branch,))
if self._userIsSure():
self.output("Deleting branch %r" % (branch,))
volumePath = self._directory.child(volume)
branchPath = volumePath.child("branches").child(branch)
branchPath.remove()
else:
self.output("Aborting.")
def _userIsSure(self):
sys.stdout.write("Are you sure (y/n)? ")
sys.stdout.flush()
return raw_input().lower() in ("y", "yes")
def setActiveVolume(self, volume):
self._directory.child(
"current_volume.json").setContent(
json.dumps(dict(current_volume=volume)))
def volume(self):
currentVolume = self._directory.child("current_volume.json")
if currentVolume.exists():
volume = json.loads(currentVolume.getContent())["current_volume"]
else:
raise UsageError("No active volume: use dvol switch to choose one")
if not self._directory.child(volume).exists():
raise UsageError("Active volume %s does not exist: "
"use dvol switch to choose another" % (volume,))
return volume
def setActiveBranch(self, volume, branch):
self._directory.child(volume).child(
"current_branch.json").setContent(
json.dumps(dict(current_branch=branch)))
self.lock.acquire(volume)
try:
self.updateRunningPoint(volume)
finally:
self.lock.release(volume)
def getActiveBranch(self, volume):
currentBranch = self._directory.child(self.volume()).child("current_branch.json")
if currentBranch.exists():
return json.loads(currentBranch.getContent())["current_branch"]
else:
return DEFAULT_BRANCH
def updateRunningPoint(self, volume):
"""
construct a stable (wrt switching branches) path with symlinks
"""
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
stablePath = volumePath.child("running_point")
if stablePath.exists():
stablePath.remove()
branchPath.linkTo(stablePath)
return stablePath.path
def commitVolume(self, message):
volume = self.volume()
commitId = (str(uuid.uuid4()) + str(uuid.uuid4())).replace("-", "")[:40]
self.output(commitId)
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
commitPath = volumePath.child("commits").child(commitId)
if commitPath.exists():
raise Exception("woah, random uuid collision. try again!")
commitPath.makedirs()
# acquire lock (read: stop containers) to ensure consistent snapshot
# with file-copy based backend
# XXX tests for acquire/release
self.lock.acquire(volume)
try:
branchPath.copyTo(commitPath)
finally:
self.lock.release(volume)
self._recordCommit(volume, branchName, commitId, message)
def _recordCommit(self, volume, branch, commitId, message):
commitData = self.commitDatabase.read(volume, branch)
commitData.append(dict(id=commitId, message=message))
self.commitDatabase.write(volume, branch, commitData)
def exists(self, volume):
volumePath = self._directory.child(volume)
return volumePath.exists()
def listVolumes(self):
table = get_table()
table.set_cols_align(["l", "l", "l"])
dc = self.lock.containers # XXX ugly
volumes = [v for v in self._directory.children() if v.isdir()]
activeVolume = None
if volumes:
try:
activeVolume = self.volume()
except UsageError:
# don't refuse to list volumes just because none of them are active
pass
rows = [["", "", ""]] + [
[" VOLUME", "BRANCH", "CONTAINERS"]] + [
[("*" if v.basename() == activeVolume else " ") + " " + v.basename(),
self.getActiveBranch(v.basename()),
",".join(c['Name'] for c in dc.get_related_containers(v.basename()))]
for v in volumes]
table.add_rows(rows)
self.output(table.draw())
def listCommits(self, branch=None):
if branch is None:
branch = self.getActiveBranch(self.volume())
volume = self.volume()
aggregate = []
for commit in reversed(self.commitDatabase.read(volume, branch)):
# TODO fill in author/date
aggregate.append(
"commit %(id)s\n"
"Author: Who knows <mystery@person>\n"
"Date: Whenever\n"
"\n"
" %(message)s\n" % commit)
self.output("\n".join(aggregate))
def _resolveNamedCommitCurrentBranch(self, commit, volume):
branch = self.getActiveBranch(volume)
remainder = commit[len("HEAD"):]
if remainder == "^" * len(remainder):
offset = len(remainder)
else:
raise UsageError("Malformed commit identifier %r" % (commit,))
commits = self.commitDatabase.read(volume, branch)
# commits are appended to, so the last one is the latest
return commits[-1 - offset]["id"]
def _destroyNewerCommits(self, commit, volume):
# TODO in the future, we'll care more about the following being an
# atomic operation
branch = self.getActiveBranch(volume)
commits = self.commitDatabase.read(volume, branch)
commitIndex = [c["id"] for c in commits].index(commit) + 1
remainingCommits = commits[:commitIndex]
destroyCommits = commits[commitIndex:]
# look in all branches for commit references before removing them
totalCommits = set()
for otherBranch in self.allBranches(volume):
if otherBranch == branch:
# skip this branch, otherwise we'll never destroy any commits
continue
commits = self.commitDatabase.read(volume, branch)
totalCommits.update(commit["id"] for commit in commits)
for commit in destroyCommits:
commitId = commit["id"]
if commitId in totalCommits:
# skip destroying this commit; it is still actively referred to
# in another branch
continue
volumePath = self._directory.child(volume)
commitPath = volumePath.child("commits").child(commitId)
commitPath.remove()
self.commitDatabase.write(volume, branch, remainingCommits)
def resetVolume(self, commit):
"""
Forcefully roll back the current working copy to this commit,
destroying any later commits.
"""
volume = self.volume()
volumePath = self._directory.child(volume)
branchName = self.getActiveBranch(volume)
branchPath = volumePath.child("branches").child(branchName)
if commit.startswith("HEAD"):
try:
commit = self._resolveNamedCommitCurrentBranch(commit, volume)
except IndexError:
self.output("Referenced commit does not exist; check dvol log")
return
commitPath = volumePath.child("commits").child(commit)
if not commitPath.exists():
raise NoSuchCommit("commit '%s' does not exist" % (commit,))
self.lock.acquire(volume)
try:
branchPath.remove()
commitPath.copyTo(branchPath)
self._destroyNewerCommits(commit, volume)
finally:
self.lock.release(volume)
verbose = False
global SCHEMADIR, PGSOCKETDIR, PSQL
for opt, arg in optargs:
if opt in ("-h", "--help"):
usage()
elif opt in ("-d",):
SCHEMADIR = arg
elif opt in ("-k",):
PGSOCKETDIR = arg
elif opt in ("-p",):
PSQL = arg
elif opt in ("-x",):
sktdir = FilePath("/var/run/caldavd")
for skt in sktdir.children():
if skt.basename().startswith("ccs_postgres_"):
PGSOCKETDIR = skt.path
PSQL = "/Applications/Server.app/Contents/ServerRoot/usr/bin/psql"
SCHEMADIR = "/Applications/Server.app/Contents/ServerRoot/Library/CalendarServer/lib/python2.7/site-packages/txdav/common/datastore/sql_schema/"
elif opt in ("-v", "--verbose"):
verbose = True
else:
raise NotImplementedError(opt)
# Retrieve the db_version number of the installed schema
try:
db_version = getSchemaVersion(verbose=verbose)
except CheckSchemaError, e:
db_version = 0
class MapUpdater(object):
def __init__(self, mapsPath, fetchURL, deleteIfNotPresent, tfLevelSounds):
assert isinstance(mapsPath, str) and len(mapsPath)
assert isinstance(fetchURL, str) and len(fetchURL)
self.mapsPath = FilePath(mapsPath)
self.downloadTempPath = self.mapsPath.child('mapupdater')
self.fetchURL = URLPath.fromString(fetchURL)
self.deleteIfNotPresent = deleteIfNotPresent
self.tfLevelSounds = tfLevelSounds
self.semaphore = DeferredSemaphore(1)
self.downloadSemaphore = DeferredSemaphore(4)
for fp in self.downloadTempPath.globChildren('*.bsp.bz2'):
fp.remove()
def checkMaps(self, *a, **kw):
"""
Wrap self._checkMaps to prevent running multiple checks at once.
"""
return self.semaphore.run(self._checkMaps, *a, **kw)
def _checkMaps(self, forceDownloadMaps=None):
def _cb(remoteMaps):
if forceDownloadMaps:
remoteMaps = list(set(remoteMaps + forceDownloadMaps))
remoteMapsLower = [f.lower() for f in remoteMaps]
ourMaps = filter(
lambda p: not p.isdir() and p.path.endswith('.bsp'),
self.mapsPath.children())
ourMapFilenames = [p.basename().lower() + '.bz2' for p in ourMaps]
missing = []
for f in remoteMaps:
if f.lower() not in ourMapFilenames:
missing.append(f)
delete = []
for p in ourMaps:
filename = p.basename().lower() + '.bz2'
if filename not in remoteMapsLower:
delete.append(p)
if self.deleteIfNotPresent and delete:
for fp in delete:
fp.remove()
print 'Deleted {} map(s) not present at remote server:'.format(
len(delete))
print ', '.join([x.basename() for x in delete])
if missing:
print 'Fetching {} map(s)'.format(len(missing))
def _allFinished(ignored):
self.mapsPath.child('tempus_map_updater_run_once').touch()
if self.tfLevelSounds:
self.addLevelSounds(ourMaps)
print 'Now up-to-date.'
ds = []
for filename in missing:
ds.append(self.fetchMap(filename))
return gatherResults(ds).addCallback(_allFinished)
elif self.tfLevelSounds:
self.addLevelSounds(ourMaps)
return self.getMapList().addCallback(_cb)
def fetchMap(self, *a, **kw):
return self.downloadSemaphore.run(self._fetchMap, *a, **kw)
def _fetchMap(self, filename):
downloadTempPath = self.downloadTempPath
if not downloadTempPath.exists():
downloadTempPath.makedirs()
def _cb(response, fn):
tp = downloadTempPath.child(fn)
fd = tp.open('wb')
def _extracted(ignored):
extractedPath = tp.sibling(tp.basename().replace('.bz2', ''))
extractedPath.moveTo(
self.mapsPath.child(tp.basename().replace('.bz2', '')))
try:
tp.remove()
# File already gone
except OSError:
pass
print 'Finished downloading {}'.format(fn)
def _finished(ignored):
fd.close()
d = getProcessOutputAndValue(
'aunpack', (tp.path, '-X', downloadTempPath.path))
d.addErrback(log.err)
d.addCallback(_extracted)
return d
def _eb(failure):
print 'Error downloading {}:'.format(fn)
print failure.getTraceback()
fd.close()
try:
tp.remove()
# File already gone
except OSError:
pass
d = treq.collect(response, fd.write)
d.addCallback(_finished)
d.addErrback(_eb)
return d
d = treq.get(str(self.fetchURL.child(filename)))
return d.addCallback(_cb, filename)
def getMapList(self, forceDownloadMaps):
raise NotImplementedError('Subclasses must override this method.')
def addLevelSounds(self, mapPaths):
content = FilePath(
mapupdater.__file__).sibling('tf_level_sounds.txt').getContent()
added = []
for p in mapPaths:
mapName = p.basename()[:-4]
p2 = p.sibling('{}_level_sounds.txt'.format(mapName))
if p2.exists() and p2.getContent() == content:
continue
added.append(mapName)
p2.setContent(content)
if added:
print 'Added level sounds for:'
print ', '.join(added)
