def test_metadata_import_export(self):
"""Two checks:
- unknown metadata is preserved across import-export
- inherited metadata doesn't leak to children.
"""
system = self.get_system()
v = 'March 20 17:00'
url_name = 'test1'
start_xml = '''
<course org="{org}" course="{course}"
due="{due}" url_name="{url_name}" unicorn="purple">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html>
</chapter>
</course>'''.format(due=v, org=ORG, course=COURSE, url_name=url_name)
descriptor = system.process_xml(start_xml)
compute_inherited_metadata(descriptor)
print(descriptor, descriptor._model_data)
self.assertEqual(descriptor.lms.due, Date().from_json(v))
# Check that the child inherits due correctly
child = descriptor.get_children()[0]
self.assertEqual(child.lms.due, Date().from_json(v))
self.assertEqual(child._inheritable_metadata, child._inherited_metadata)
self.assertEqual(2, len(child._inherited_metadata))
self.assertEqual('1970-01-01T00:00:00Z', child._inherited_metadata['start'])
self.assertEqual(v, child._inherited_metadata['due'])
# Now export and check things
resource_fs = MemoryFS()
exported_xml = descriptor.export_to_xml(resource_fs)
# Check that the exported xml is just a pointer
print("Exported xml:", exported_xml)
pointer = etree.fromstring(exported_xml)
self.assertTrue(is_pointer_tag(pointer))
# but it's a special case course pointer
self.assertEqual(pointer.attrib['course'], COURSE)
self.assertEqual(pointer.attrib['org'], ORG)
# Does the course still have unicorns?
with resource_fs.open('course/{url_name}.xml'.format(url_name=url_name)) as f:
course_xml = etree.fromstring(f.read())
self.assertEqual(course_xml.attrib['unicorn'], 'purple')
# the course and org tags should be _only_ in the pointer
self.assertTrue('course' not in course_xml.attrib)
self.assertTrue('org' not in course_xml.attrib)
# did we successfully strip the url_name from the definition contents?
self.assertTrue('url_name' not in course_xml.attrib)
# Does the chapter tag now have a due attribute?
# hardcoded path to child
with resource_fs.open('chapter/ch.xml') as f:
chapter_xml = etree.fromstring(f.read())
self.assertEqual(chapter_xml.tag, 'chapter')
self.assertFalse('due' in chapter_xml.attrib)
def test_writefs_returns_none_if_all_fs_closed(self):
# Arrange
multifs = WritableMultiFS()
fs1 = MemoryFS()
multifs.addfs("fs1", fs1)
fs1.close()
# Act
assert multifs.writefs is None
def test_free_space_returns_meta_if_has_meta(self):
# Arrange
fs = MemoryFS()
fs.getmeta = Mock(return_value=mb(230))
# Act
space = free_space(fs)
# Assert
assert space == mb(230)
def test_safety_wrapper(self):
rawfs = MemoryFS()
safefs = dokan.Win32SafetyFS(rawfs)
rawfs.setcontents("autoRun.inf", b("evilcodeevilcode"))
self.assertTrue(safefs.exists("_autoRun.inf"))
self.assertTrue("autoRun.inf" not in safefs.listdir("/"))
safefs.setcontents("file:stream",b("test"))
self.assertFalse(rawfs.exists("file:stream"))
self.assertTrue(rawfs.exists("file__colon__stream"))
self.assertTrue("file:stream" in safefs.listdir("/"))
def check_property(self, descriptor):
xmodule_api_fs = MemoryFS()
xblock_api_fs = MemoryFS()
descriptor.runtime.export_fs = xblock_api_fs
xblock_node = etree.Element('unknown')
descriptor.add_xml_to_node(xblock_node)
xmodule_node = etree.fromstring(descriptor.export_to_xml(xmodule_api_fs))
self.assertEquals(list(xmodule_api_fs.walk()), list(xblock_api_fs.walk()))
self.assertEquals(etree.tostring(xmodule_node), etree.tostring(xblock_node))
class TestFileCache(unittest.TestCase, CacheTests, NamespacesTests):
__test__ = True
def setUp(self):
self.fs = MemoryFS()
self.cache = cache.filecache.FileCache("test", "ns1", fs=self.fs)
self.cache2 = cache.filecache.FileCache("test", "ns2", fs=self.fs)
def tearDown(self):
self.fs.close()
self.fs = None
def test_copydir_indir(self):
"""Test copydir in a directory"""
fs1 = MemoryFS()
fs2 = MemoryFS()
self._make_fs(fs1)
utils.copydir(fs1, (fs2, "copy"))
self._check_fs(fs2.opendir("copy"))
fs1 = TempFS()
fs2 = TempFS()
self._make_fs(fs1)
utils.copydir(fs1, (fs2, "copy"))
self._check_fs(fs2.opendir("copy"))
def test_movedir_indir(self):
"""Test movedir in a directory"""
fs1 = MemoryFS()
fs2 = MemoryFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), (fs2, "copy"))
self.assert_(not fs1.exists("from"))
self._check_fs(fs2.opendir("copy"))
fs1 = TempFS()
fs2 = TempFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), (fs2, "copy"))
self.assert_(not fs1.exists("from"))
self._check_fs(fs2.opendir("copy"))
def test_mountfile(self):
"""Test mounting a file"""
quote = b"""If you wish to make an apple pie from scratch, you must first invent the universe."""
mem_fs = MemoryFS()
mem_fs.makedir('foo')
mem_fs.setcontents('foo/bar.txt', quote)
foo_dir = mem_fs.opendir('foo')
mount_fs = MountFS()
mount_fs.mountfile('bar.txt', foo_dir.open, foo_dir.getinfo)
self.assert_(mount_fs.isdir('/'))
self.assert_(mount_fs.isdir('./'))
self.assert_(mount_fs.isdir(''))
# Check we can see the mounted file in the dir list
self.assertEqual(mount_fs.listdir(), ["bar.txt"])
self.assert_(not mount_fs.exists('nobodyhere.txt'))
self.assert_(mount_fs.exists('bar.txt'))
self.assert_(mount_fs.isfile('bar.txt'))
self.assert_(not mount_fs.isdir('bar.txt'))
# Check open and getinfo callables
self.assertEqual(mount_fs.getcontents('bar.txt'), quote)
self.assertEqual(mount_fs.getsize('bar.txt'), len(quote))
# Check changes are written back
mem_fs.setcontents('foo/bar.txt', 'baz')
self.assertEqual(mount_fs.getcontents('bar.txt'), b'baz')
self.assertEqual(mount_fs.getsize('bar.txt'), len('baz'))
# Check changes are written to the original fs
self.assertEqual(mem_fs.getcontents('foo/bar.txt'), b'baz')
self.assertEqual(mem_fs.getsize('foo/bar.txt'), len('baz'))
# Check unmount
self.assert_(mount_fs.unmount("bar.txt"))
self.assertEqual(mount_fs.listdir(), [])
self.assert_(not mount_fs.exists('bar.txt'))
# Check unount a second time is a null op, and returns False
self.assertFalse(mount_fs.unmount("bar.txt"))
class TestWalk(unittest.TestCase):
def setUp(self):
self.fs = MemoryFS()
self.fs.setcontents('a.txt', 'hello')
self.fs.setcontents('b.txt', 'world')
self.fs.makeopendir('foo').setcontents('c', '123')
self.fs.makeopendir('.svn').setcontents('ignored', '')
def test_wildcard(self):
for dir_path, paths in self.fs.walk(wildcard='*.txt'):
for path in paths:
self.assert_(path.endswith('.txt'))
for dir_path, paths in self.fs.walk(wildcard=lambda fn:fn.endswith('.txt')):
for path in paths:
self.assert_(path.endswith('.txt'))
def test_dir_wildcard(self):
for dir_path, paths in self.fs.walk(dir_wildcard=lambda fn:not fn.endswith('.svn')):
for path in paths:
self.assert_('.svn' not in path)
def __init__(self, root, cmd_channel):
AbstractedFS.__init__(self, root, cmd_channel)
self.cwd = root
self.type = cmd_channel.type
self.s3_bucket = cmd_channel.s3_bucket
self.aws_access_key = cmd_channel.aws_access_key
self.aws_secret_key = cmd_channel.aws_secret_key
self.seperator = cmd_channel.seperator
self.thread_synchronize = cmd_channel.thread_synchronize
self.key_sync_timeout = cmd_channel.key_sync_timeout
if not self.cmd_channel.fs_obj:
if self.type == "memory":
self.fs_obj = MemoryFS()
elif self.type == "s3":
self.fs_obj = S3FS(bucket=self.bucket, prefix=self.prefix, aws_access_key=self.aws_access_key, aws_secret_key=self.aws_secret_key, separator=self.seperator, thread_synchronize=self.thread_synchronize, key_sync_timeout=self.key_sync_timeout)
self.cmd_channel.fs_obj = self.fs_obj
else:
self.fs_obj = self.cmd_channel.fs_obj
def test_copy_files(self):
mfs = MemoryFS()
mfs.touch('txt1.txt')
mfs.touch('txt2.txt')
mfs.makedir('sub1')
mfs.makedir('sub2')
new_files = [
'sub1/txt1.txt', 'sub1/txt2.txt', 'sub2/txt1.txt', 'sub2/txt2.txt'
]
for n in new_files:
self.assertFalse(mfs.exists(n))
d = Directory('.', mfs)
targets = d.listdir_as_observable().filter(match_directory(['sub*']))
sources = d.listdir_as_observable().filter(match_file(['txt*']))
sources.subscribe(lambda f: print(f.path.s))
sources_list = []
sources.subscribe(sources_list.append)
results = (targets.flat_map(
lambda d: d.sync(sources)).to_list().to_blocking().first())
self.assertEqual(len(results), 4)
for n in new_files:
self.assertTrue(mfs.exists(n))
def setUp(self):
super(BaseVerticalBlockTest, self).setUp()
# construct module
course = xml.CourseFactory.build()
sequence = xml.SequenceFactory.build(parent=course)
vertical = xml.VerticalFactory.build(parent=sequence)
self.course = self.process_xml(course)
xml.HtmlFactory(parent=vertical, url_name='test-html-1', text=self.test_html_1)
xml.HtmlFactory(parent=vertical, url_name='test-html-2', text=self.test_html_2)
self.course = self.process_xml(course)
course_seq = self.course.get_children()[0]
self.module_system = get_test_system()
self.module_system.descriptor_runtime = self.course._runtime # pylint: disable=protected-access
self.course.runtime.export_fs = MemoryFS()
self.vertical = course_seq.get_children()[0]
self.vertical.xmodule_runtime = self.module_system
def test_export_import_round_trip(self, def_to_xml):
# The HtmlDescriptor definition_to_xml tries to write to the filesystem
# before returning an xml object. Patch this to just return the xml.
def_to_xml.return_value = lxml.etree.Element('html')
# Mock out the process_xml
# Expect it to return a child descriptor for the SplitTestDescriptor when called.
self.module_system.process_xml = Mock()
# Write out the xml.
xml_obj = self.split_test_module.definition_to_xml(MemoryFS())
self.assertEquals(xml_obj.get('user_partition_id'), '0')
self.assertIsNotNone(xml_obj.get('group_id_to_child'))
# Read the xml back in.
fields, children = SplitTestDescriptor.definition_from_xml(xml_obj, self.module_system)
self.assertEquals(fields.get('user_partition_id'), '0')
self.assertIsNotNone(fields.get('group_id_to_child'))
self.assertEquals(len(children), 2)
def check_property(self, descriptor):
xmodule_api_fs = MemoryFS()
xblock_api_fs = MemoryFS()
descriptor.runtime.export_fs = xblock_api_fs
xblock_node = etree.Element('unknown')
descriptor.add_xml_to_node(xblock_node)
xmodule_node = etree.fromstring(
descriptor.export_to_xml(xmodule_api_fs))
self.assertEquals(list(xmodule_api_fs.walk()),
list(xblock_api_fs.walk()))
self.assertEquals(etree.tostring(xmodule_node),
etree.tostring(xblock_node))
def __init__(self, slf_filename):
super(SlfFS, self).__init__()
if isinstance(slf_filename, str):
slf_filename = os.path.expanduser(os.path.expandvars(slf_filename))
slf_filename = os.path.normpath(os.path.abspath(slf_filename))
try:
self.file_name = slf_filename
self.file = open(slf_filename, 'rb')
except FileNotFoundError as e:
raise CreateFailedError(
'Slf file not found ({0})'.format(slf_filename),
details=e
)
else:
self.file_name = 'file-like'
self.file = slf_filename
self.header = SlfHeader.from_bytes(self.file.read(SlfHeader.get_size()))
self.entries = list(map(self._read_entry, range(self.header['number_of_entries'])))
self.library_name = self.header['library_name']
self.library_path = self.header['library_path']
self.sort = self.header['sort']
self.version = self.header['version']
self._path_fs = MemoryFS()
for e in self.entries:
path = _get_normalized_filename(e['file_name']).split('/')
directory = '/'.join(path[:-1]) if len(path) > 2 else '/'
if self._path_fs.isfile(directory):
# Sometimes there exists a file that has the same name as a directory
# Solution: Rename it with a _DIRECTORY_CONFLICT suffix
self._path_fs.move(directory, directory + DIRECTORY_CONFLICT_SUFFIX)
if self._path_fs.isdir('/'.join(path)):
self._path_fs.createfile('/'.join(path) + DIRECTORY_CONFLICT_SUFFIX)
else:
self._path_fs.makedir(directory, recursive=True, allow_recreate=True)
self._path_fs.createfile('/'.join(path))
def _create(
self,
fs: Optional[FS],
name: Optional[str],
id_: Optional[str],
metadata: Optional[dict],
combinatorial: bool = False,
sequential: bool = False,
seed: Optional[int] = None,
sum_intra_duplicates: bool = True,
sum_inter_duplicates: bool = False,
) -> None:
"""Start a new data package.
All metadata elements should follow the `datapackage specification <https://frictionlessdata.io/specs/data-package/>`__.
Licenses are specified as a list in ``metadata``. The default license is the `Open Data Commons Public Domain Dedication and License v1.0 <http://opendatacommons.org/licenses/pddl/>`__.
"""
name = clean_datapackage_name(name or uuid.uuid4().hex)
check_name(name)
self.fs = fs or MemoryFS()
self.metadata = {
"profile": "data-package",
"name": name,
"id": id_ or uuid.uuid4().hex,
"licenses": (metadata or {}).get("licenses", DEFAULT_LICENSES),
"resources": [],
"created": datetime.datetime.utcnow().isoformat("T") + "Z",
"combinatorial": combinatorial,
"sequential": sequential,
"seed": seed,
"sum_intra_duplicates": sum_intra_duplicates,
"sum_inter_duplicates": sum_inter_duplicates,
}
for k, v in (metadata or {}).items():
if k not in self.metadata:
self.metadata[k] = v
self.data = []
def test__when_copying_dirs_with_glob_patterns__it_copies_matching_dirs_with_content(
):
mem_fs = MemoryFS()
sub_fs = mem_fs.makedirs("sub/first")
sub_fs.create("file.txt")
sub_fs = mem_fs.makedirs("sub/second")
sub_fs.create("another.txt")
sut = _TestFilesystemImpl(mem_fs)
sut.copy("sub/*", "otherdir/")
assert mem_fs.exists("otherdir/first/file.txt")
assert mem_fs.exists("otherdir/second/another.txt")
class DummySystem(ImportSystem):
@patch('xmodule.modulestore.xml.OSFS', lambda dir: MemoryFS())
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir",
course_dirs=[],
load_error_modules=load_error_modules)
course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run')
course_dir = "test_dir"
error_tracker = Mock()
parent_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
parent_tracker=parent_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
)
class DummySystem(ImportSystem): # lint-amnesty, pylint: disable=abstract-method, missing-class-docstring
@patch('xmodule.modulestore.xml.OSFS', lambda dir: MemoryFS())
def __init__(self, load_error_modules, course_id=None):
xmlstore = XMLModuleStore("data_dir",
source_dirs=[],
load_error_modules=load_error_modules)
if course_id is None:
course_id = CourseKey.from_string('/'.join(
[ORG, COURSE, 'test_run']))
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__( # lint-amnesty, pylint: disable=super-with-arguments
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
field_data=KvsFieldData(DictKeyValueStore()),
)
class DummySystem(ImportSystem):
@patch('xmodule.modulestore.xml.OSFS', lambda dir: MemoryFS())
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir", course_dirs=[],
load_error_modules=load_error_modules)
course_id = "/".join([ORG, COURSE, 'test_run'])
course_dir = "test_dir"
policy = {}
error_tracker = Mock()
parent_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore,
course_id,
course_dir,
policy,
error_tracker,
parent_tracker,
load_error_modules=load_error_modules,
)
def test__when_copying_directory_to_other_filesystem__should_copy_dir():
source_dir = "mydir"
target_dir = "copydir"
origin_fs = MemoryFS()
sub_fs = origin_fs.makedir(source_dir)
write_file_with_content(sub_fs, SOURCE, "content")
sut = _TestFilesystemImpl(origin_fs)
target_fs = MemoryFS()
sut.copy(source_dir, target_dir, filesystem=_TestFilesystemImpl(target_fs))
complete_path = f"{target_dir}/{SOURCE}"
assert target_fs.exists(complete_path)
assert_file_content_equals(target_fs, complete_path, "content")
def test_apply(self):
mfs = MemoryFS()
subs = ['sub.{}'.format(i) for i in range(3)]
for d in subs:
mfs.makedir(d)
d = Directory('.', mfs)
mfs.touch('test1.txt')
mfs.touch('test2.txt')
o0 = ini.OpAddToBroadcastFile('test1.txt')
o1 = ini.OpAddToBroadcastFile('test2.txt')
obc = ini.OpBroadcastFile(['sub*'])
r = RoutineOnDirectory(d, [o0, o1, obc])
r.work()
target_files = []
for d in subs:
for f in ['test1.txt', 'test2.txt']:
target_files.append(d + '/' + f)
for i, t in enumerate(target_files):
with self.subTest(i):
self.assertTrue(mfs.exists(t))
def test_movedir_root(self):
"""Test movedir to root dir"""
fs1 = MemoryFS()
fs2 = MemoryFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), fs2)
self.assert_(not fs1.exists("from"))
self._check_fs(fs2)
fs1 = TempFS()
fs2 = TempFS()
fs1sub = fs1.makeopendir("from")
self._make_fs(fs1sub)
utils.movedir((fs1, "from"), fs2)
self.assert_(not fs1.exists("from"))
self._check_fs(fs2)
def override_export_fs(block):
"""
Hack required for some legacy XBlocks which inherit
XModuleDescriptor.add_xml_to_node()
instead of the usual
XmlSerializationMixin.add_xml_to_node()
method.
This method temporarily replaces a block's runtime's 'export_fs' system with
an in-memory filesystem. This method also abuses the
XmlParserMixin.export_to_file()
API to prevent the XModule export code from exporting each block as two
files (one .olx pointing to one .xml file). The export_to_file was meant to
be used only by the customtag XModule but it makes our lives here much
easier.
"""
fs = WrapFS(MemoryFS())
fs.makedir('course')
fs.makedir(
'course/static'
) # Video XBlock requires this directory to exists, to put srt files etc.
old_export_fs = block.runtime.export_fs
block.runtime.export_fs = fs
if hasattr(block, 'export_to_file'):
old_export_to_file = block.export_to_file
block.export_to_file = lambda: False
old_global_export_to_file = XmlParserMixin.export_to_file
XmlParserMixin.export_to_file = lambda _: False # So this applies to child blocks that get loaded during export
try:
yield fs
except:
raise
finally:
block.runtime.export_fs = old_export_fs
if hasattr(block, 'export_to_file'):
block.export_to_file = old_export_to_file
XmlParserMixin.export_to_file = old_global_export_to_file
class DummySystem(ImportSystem):
@patch('xmodule.modulestore.xml.OSFS', lambda dir: MemoryFS())
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir",
course_dirs=[],
load_error_modules=load_error_modules)
course_id = SlashSeparatedCourseKey(ORG, COURSE, 'test_run')
course_dir = "test_dir"
error_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore=xmlstore,
course_id=course_id,
course_dir=course_dir,
error_tracker=error_tracker,
load_error_modules=load_error_modules,
mixins=(InheritanceMixin, XModuleMixin),
field_data=KvsFieldData(DictKeyValueStore()),
)
def render_template(self, _template, _context):
raise Exception("Shouldn't be called")
def setUp(self):
super().setUp()
# construct module: course/sequence/vertical - problems
# \_ nested_vertical / problems
course = xml.CourseFactory.build()
sequence = xml.SequenceFactory.build(parent=course)
vertical = xml.VerticalFactory.build(parent=sequence)
self.course = self.process_xml(course)
xml.HtmlFactory(parent=vertical, url_name='test-html', text=self.test_html)
xml.ProblemFactory(parent=vertical, url_name='test-problem', text=self.test_problem)
nested_vertical = xml.VerticalFactory.build(parent=vertical)
xml.HtmlFactory(parent=nested_vertical, url_name='test_html_nested', text=self.test_html_nested)
xml.ProblemFactory(parent=nested_vertical, url_name='test_problem_nested', text=self.test_problem_nested)
self.course = self.process_xml(course)
course_seq = self.course.get_children()[0]
self.module_system = get_test_system()
self.module_system.descriptor_runtime = self.course._runtime
self.course.runtime.export_fs = MemoryFS()
self.vertical = course_seq.get_children()[0]
self.vertical.xmodule_runtime = self.module_system
self.html_block = self.vertical.get_children()[0]
self.problem_block = self.vertical.get_children()[1]
self.problem_block.has_score = True
self.problem_block.graded = True
self.extra_vertical_block = self.vertical.get_children()[2] # VerticalBlockWithMixins
self.nested_problem_block = self.extra_vertical_block.get_children()[1]
self.nested_problem_block.has_score = True
self.nested_problem_block.graded = True
self.username = "******"
self.default_context = {"bookmarked": False, "username": self.username}
def test_listdir(self):
mfs = MemoryFS()
mfs.touch('test_file.txt')
mfs.makedir('test_dir')
d = Directory('.', mfs)
result = d.listdir()
self.assertEqual(len(result), 2)
cdir = None
cfile = None
for o in result:
if isinstance(o, Directory):
cdir = o
else:
cfile = o
self.assertIsNotNone(cdir)
self.assertIsNotNone(cfile)
self.assertEqual(cdir.path.n, 'test_dir')
self.assertEqual(cfile.path.n, 'test_file.txt')
class DummySystem(ImportSystem):
@patch('xmodule.modulestore.xml.OSFS', lambda directory: MemoryFS())
def __init__(self, load_error_modules):
xmlstore = XMLModuleStore("data_dir", course_dirs=[], load_error_modules=load_error_modules)
course_id = "/".join([ORG, COURSE, 'test_run'])
course_dir = "test_dir"
policy = {}
error_tracker = Mock()
parent_tracker = Mock()
super(DummySystem, self).__init__(
xmlstore,
course_id,
course_dir,
policy,
error_tracker,
parent_tracker,
load_error_modules=load_error_modules,
)
def render_template(self, template, context):
raise Exception("Shouldn't be called")
def __init__(self, filename, mode="r", thread_synchronize=True):
"""Create a FS that maps on to a big file.
:param filename: A (system) path, or a file-like object
:param mode: Mode to open file: 'r' for reading, 'w' and 'a' not supported
:param thread_synchronize: -- Set to True (default) to enable thread-safety
"""
super(BigFS, self).__init__(thread_synchronize=thread_synchronize)
if len(mode) > 1 or mode not in "r":
raise ValueError("mode must be 'r'")
self.file_mode = mode
self.big_path = str(filename)
self.entries = {}
try:
self.bf = open(filename, "rb")
except IOError:
raise ResourceNotFoundError(str(filename), msg="BIG file does not exist: %(path)s")
self._path_fs = MemoryFS()
if mode in 'ra':
self._parse_resource_list(self.bf)
path = relpath(normpath(path))
path = path.replace("__colon__", ":")
if not self.allow_autorun:
if path.lower().startswith("autorun."):
path = "_" + path
return path
if __name__ == "__main__":
import os.path
import tempfile
from fs.osfs import OSFS
from fs.memoryfs import MemoryFS
from shutil import rmtree
from six import b
path = tempfile.mkdtemp()
try:
#fs = OSFS(path)
fs = MemoryFS()
fs.create('test.txt')
fs.appendtext('test.txt',
'this is a test',
encoding=u'utf-8',
errors=None,
newline=u'')
flags = DOKAN_OPTION_DEBUG | DOKAN_OPTION_STDERR | DOKAN_OPTION_REMOVABLE
mount(fs, "Q:\\", foreground=True, numthreads=1, flags=flags)
fs.close()
finally:
rmtree(path)
This
is a test file
{{%- if readme %}}
@readme.txt
Readme file
-----------
${{message}}
{{%- endif %}}
@templates/base.html
<h1>${title}</h1>
<ul>
{% for fruit in fruits %}
<li>${fruit}</li>
{% endfor %}
</ul>
@settings/production.ini
@foo/bar/baz/
@author
Bob
"""
from fs.osfs import OSFS
from fs.memoryfs import MemoryFS
fs = OSFS('__test__', create=True)
fs = MemoryFS()
td = dict(message="Hello, World!", readme=True)
compile_fs_template(fs, template, td)
fs.tree()
class SlfFS(FS):
"""
Implements a read-only file system on top of a SLF-file
"""
_meta = {
'thread_safe': False,
'virtual': False,
'read_only': True,
'unicode_paths': False,
'case_insensitive_paths': False,
'network': False,
'atomic.setcontents': False
}
def __init__(self, slf_filename):
super(SlfFS, self).__init__()
if isinstance(slf_filename, str):
slf_filename = os.path.expanduser(os.path.expandvars(slf_filename))
slf_filename = os.path.normpath(os.path.abspath(slf_filename))
try:
self.file_name = slf_filename
self.file = open(slf_filename, 'rb')
except FileNotFoundError as e:
raise CreateFailedError(
'Slf file not found ({0})'.format(slf_filename),
details=e
)
else:
self.file_name = 'file-like'
self.file = slf_filename
self.header = SlfHeader.from_bytes(self.file.read(SlfHeader.get_size()))
self.entries = list(map(self._read_entry, range(self.header['number_of_entries'])))
self.library_name = self.header['library_name']
self.library_path = self.header['library_path']
self.sort = self.header['sort']
self.version = self.header['version']
self._path_fs = MemoryFS()
for e in self.entries:
path = _get_normalized_filename(e['file_name']).split('/')
directory = '/'.join(path[:-1]) if len(path) > 2 else '/'
if self._path_fs.isfile(directory):
# Sometimes there exists a file that has the same name as a directory
# Solution: Rename it with a _DIRECTORY_CONFLICT suffix
self._path_fs.move(directory, directory + DIRECTORY_CONFLICT_SUFFIX)
if self._path_fs.isdir('/'.join(path)):
self._path_fs.createfile('/'.join(path) + DIRECTORY_CONFLICT_SUFFIX)
else:
self._path_fs.makedir(directory, recursive=True, allow_recreate=True)
self._path_fs.createfile('/'.join(path))
def _read_entry(self, index):
entry_size = SlfEntry.get_size()
self.file.seek(-entry_size * (self.header['number_of_entries'] - index), os.SEEK_END)
return SlfEntry.from_bytes(self.file.read(entry_size))
def __str__(self):
return '<SlfFS: {0}>'.format(self['library_name'])
def isfile(self, path):
return self._path_fs.isfile(path)
def isdir(self, path):
return self._path_fs.isdir(path)
def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only)
def open(self, path, mode='r', buffering=-1, encoding='ascii', errors=None, newline=None, line_buffering=False, **kwargs):
if mode != 'r' and mode != 'rb':
raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('open'))
if not self.exists(path):
raise ResourceNotFoundError(path)
if self.isdir(path):
raise ResourceInvalidError(path)
slf_entry = self._get_slf_entry_for_path(path)
self.file.seek(slf_entry['offset'], os.SEEK_SET)
if mode == 'rb':
return io.BytesIO(self.file.read(slf_entry['length']))
return io.StringIO(self.file.read(slf_entry['length']).decode(encoding))
def getinfo(self, path):
if not self.exists(path):
raise ResourceNotFoundError(path)
if self.isdir(path):
return {
'size': 0
}
slf_entry = self._get_slf_entry_for_path(path)
return {
'size': slf_entry['length'],
'modified_time': slf_entry['time']
}
def makedir(self, path, recursive=False, allow_recreate=False):
raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('makedir'))
def remove(self, path):
raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('remove'))
def removedir(self, path, recursive=False, force=False):
raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('removedir'))
def rename(self, src, dst):
raise UnsupportedError(WRITING_NOT_SUPPORTED_ERROR.format('rename'))
def _get_slf_entry_for_path(self, path):
if path.endswith(DIRECTORY_CONFLICT_SUFFIX):
path = path[:-len(DIRECTORY_CONFLICT_SUFFIX)]
return next(e for e in self.entries if _get_normalized_filename(e['file_name']) == path)
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir):
from fs.memoryfs import MemoryFS
memfs = MemoryFS()
if create_dir:
memfs = memfs.makeopendir(fs_path)
return memfs, None
def test_priority(self):
"""Test priority order is working"""
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.writebytes("name", b"m1")
m2.writebytes("name", b"m2")
m3.writebytes("name", b"m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.add_fs("m1", m1)
multi_fs.add_fs("m2", m2)
multi_fs.add_fs("m3", m3)
self.assertEqual(multi_fs.readbytes("name"), b"m3")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.writebytes("name", b"m1")
m2.writebytes("name", b"m2")
m3.writebytes("name", b"m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.add_fs("m1", m1)
multi_fs.add_fs("m2", m2, priority=10)
multi_fs.add_fs("m3", m3)
self.assertEqual(multi_fs.readbytes("name"), b"m2")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.writebytes("name", b"m1")
m2.writebytes("name", b"m2")
m3.writebytes("name", b"m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.add_fs("m1", m1)
multi_fs.add_fs("m2", m2, priority=10)
multi_fs.add_fs("m3", m3, priority=10)
self.assertEqual(multi_fs.readbytes("name"), b"m3")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.writebytes("name", b"m1")
m2.writebytes("name", b"m2")
m3.writebytes("name", b"m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.add_fs("m1", m1, priority=11)
multi_fs.add_fs("m2", m2, priority=10)
multi_fs.add_fs("m3", m3, priority=10)
self.assertEqual(multi_fs.readbytes("name"), b"m1")
def test_get_set_items(self) -> None:
no_lidvids: Set[LIDVID] = set()
# Set empty contents
mv = Multiversioned(MemoryFS())
empty_lidvid = LIDVID("urn:nasa:pds:empty-bundle::3.14")
empty_contents = VersionContents.create_from_lidvids(
no_lidvids, MemoryFS(), set())
mv[empty_lidvid] = empty_contents
self.assertTrue(empty_lidvid in mv)
self.assertEqual(1, len(mv))
self.assertEqual(empty_contents, mv[empty_lidvid])
# TODO Is IndexError the right exception to raise?
with self.assertRaises(IndexError):
mv[empty_lidvid] = empty_contents
# Set contents with a single file down a long path
single_file_lidvid = LIDVID("urn:nasa:pds:single-file::3.14")
single_file_fs = MemoryFS()
single_file_path = "/down/a/lot/of/dirs/text.txt"
single_file_fs.makedirs(fs.path.dirname(single_file_path), None, True)
single_file_fs.writetext(single_file_path, "Hello, there!")
single_file_contents = VersionContents.create_from_lidvids(
no_lidvids, single_file_fs, set([single_file_path]))
mv[single_file_lidvid] = single_file_contents
self.assertTrue(empty_lidvid in mv)
self.assertTrue(single_file_lidvid in mv)
print("****", set(mv.lidvids()))
self.assertEqual(2, len(mv))
self.assertEqual(single_file_contents, mv[single_file_lidvid])
# Test that LIDVIDs get put correctly into the
# subdir$version.txts.
hierarchic = Multiversioned(MemoryFS())
b_lidvid = LIDVID("urn:nasa:pds:b::1.5")
c_lidvid = LIDVID("urn:nasa:pds:b:c::2.5")
p_lidvid = LIDVID("urn:nasa:pds:b:c:p::333.123")
p_contents = VersionContents.create_from_lidvids(
no_lidvids, MemoryFS(), set([]))
hierarchic[p_lidvid] = p_contents
self.assertEqual(p_contents, hierarchic[p_lidvid])
c_contents = VersionContents.create_from_lidvids(
set([p_lidvid]), MemoryFS(), set([]))
hierarchic[c_lidvid] = c_contents
self.assertEqual(c_contents, hierarchic[c_lidvid])
b_contents = VersionContents.create_from_lidvids(
set([c_lidvid]), MemoryFS(), set([]))
hierarchic[b_lidvid] = b_contents
self.assertEqual(b_contents, hierarchic[b_lidvid])
self.assertEqual(3, len(hierarchic))
self.assertEqual({p_lidvid, c_lidvid, b_lidvid}, hierarchic.lidvids())
def setUp(self):
self.fs = MemoryFS()
def setUp(self):
self.fs = MemoryFS()
self.fs.setcontents('a.txt', 'hello')
self.fs.setcontents('b.txt', 'world')
self.fs.makeopendir('foo').setcontents('c', '123')
self.fs.makeopendir('.svn').setcontents('ignored', '')
def test_priority(self):
"""Test priority order is working"""
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2)
multi_fs.addfs("m3", m3)
self.assert_(multi_fs.getcontents("name") == "m3")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3)
self.assert_(multi_fs.getcontents("name") == "m2")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3, priority=10)
self.assert_(multi_fs.getcontents("name") == "m3")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1, priority=11)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3, priority=10)
self.assert_(multi_fs.getcontents("name") == "m1")
class TestCaseWithMemoryFS(unittest.TestCase):
def setUp(self):
self.fs = MemoryFS()
def tearDown(self):
self.fs.close()
def test_metadata_import_export(self):
"""Two checks:
- unknown metadata is preserved across import-export
- inherited metadata doesn't leak to children.
"""
system = self.get_system()
v = 'March 20 17:00'
url_name = 'test1'
start_xml = '''
<course org="{org}" course="{course}"
due="{due}" url_name="{url_name}" unicorn="purple">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html>
</chapter>
</course>'''.format(due=v, org=ORG, course=COURSE, url_name=url_name)
descriptor = system.process_xml(start_xml)
compute_inherited_metadata(descriptor)
print(descriptor, descriptor._model_data)
self.assertEqual(descriptor.lms.due, Date().from_json(v))
# Check that the child inherits due correctly
child = descriptor.get_children()[0]
self.assertEqual(child.lms.due, Date().from_json(v))
self.assertEqual(child._inheritable_metadata,
child._inherited_metadata)
self.assertEqual(2, len(child._inherited_metadata))
self.assertEqual('1970-01-01T00:00:00Z',
child._inherited_metadata['start'])
self.assertEqual(v, child._inherited_metadata['due'])
# Now export and check things
resource_fs = MemoryFS()
exported_xml = descriptor.export_to_xml(resource_fs)
# Check that the exported xml is just a pointer
print("Exported xml:", exported_xml)
pointer = etree.fromstring(exported_xml)
self.assertTrue(is_pointer_tag(pointer))
# but it's a special case course pointer
self.assertEqual(pointer.attrib['course'], COURSE)
self.assertEqual(pointer.attrib['org'], ORG)
# Does the course still have unicorns?
with resource_fs.open(
'course/{url_name}.xml'.format(url_name=url_name)) as f:
course_xml = etree.fromstring(f.read())
self.assertEqual(course_xml.attrib['unicorn'], 'purple')
# the course and org tags should be _only_ in the pointer
self.assertTrue('course' not in course_xml.attrib)
self.assertTrue('org' not in course_xml.attrib)
# did we successfully strip the url_name from the definition contents?
self.assertTrue('url_name' not in course_xml.attrib)
# Does the chapter tag now have a due attribute?
# hardcoded path to child
with resource_fs.open('chapter/ch.xml') as f:
chapter_xml = etree.fromstring(f.read())
self.assertEqual(chapter_xml.tag, 'chapter')
self.assertFalse('due' in chapter_xml.attrib)
class BigFS(FS):
"""A FileSystem that represents a BIG file."""
_meta = { 'virtual' : False,
'read_only' : True,
'unicode_paths' : True,
'case_insensitive_paths' : False,
'network' : False,
}
def __init__(self, filename, mode="r", thread_synchronize=True):
"""Create a FS that maps on to a big file.
:param filename: A (system) path, or a file-like object
:param mode: Mode to open file: 'r' for reading, 'w' and 'a' not supported
:param thread_synchronize: -- Set to True (default) to enable thread-safety
"""
super(BigFS, self).__init__(thread_synchronize=thread_synchronize)
if len(mode) > 1 or mode not in "r":
raise ValueError("mode must be 'r'")
self.file_mode = mode
self.big_path = str(filename)
self.entries = {}
try:
self.bf = open(filename, "rb")
except IOError:
raise ResourceNotFoundError(str(filename), msg="BIG file does not exist: %(path)s")
self._path_fs = MemoryFS()
if mode in 'ra':
self._parse_resource_list(self.bf)
def __str__(self):
return "<BigFS: %s>" % self.big_path
def __unicode__(self):
return unicode(self.__str__())
def _parse_resource_list(self, g):
magicWord = g.read(4)
if magicWord != "BIGF" and magicWord != "BIG4":
raise ValueError("Magic word of BIG file invalid: " + filename + " " + repr(magicWord))
header = g.read(12)
header = unpack(">III", header)
BIGSize = header[0]
fileCount = header[1]
bodyOffset = header[2]
for i in range(fileCount):
fileHeader = g.read(8)
fileHeader = unpack(">II", fileHeader)
pos = g.tell()
buf = g.read(4096)
marker = buf.find("\0")
if marker == -1:
raise ValueError("Could not parse filename in BIG file: Too long or invalid file")
name = buf[:marker]
# TODO: decode the encoding of name (or normalize the path?)
isCompressed, uncompressedSize = self.__isCompressed(g, fileHeader[0], fileHeader[1])
be = BIGEntry(name, fileHeader[0], fileHeader[1], isCompressed, uncompressedSize)
name = normpath(name)
self.entries[name] = be
self._add_resource(name)
g.seek(pos + marker + 1)
def __isCompressed(self, g, offset, size):
g.seek(offset)
buf = g.read(2)
magic = unpack(">H", buf)[0]
if (magic & 0x3EFF) == 0x10FB:
# it is compressed
if magic & 0x8000:
# decompressed size is uint32
return True, unpack(">I", g.read(4))[0]
else:
# use only 3 bytes
return True, unpack(">I", "\0" + g.read(3))[0]
return False, size
def _add_resource(self, path):
if path.endswith('/'):
path = path[:-1]
if path:
self._path_fs.makedir(path, recursive=True, allow_recreate=True)
else:
dirpath, filename = pathsplit(path)
if dirpath:
self._path_fs.makedir(dirpath, recursive=True, allow_recreate=True)
f = self._path_fs.open(path, 'w')
f.close()
def close(self):
"""Finalizes the zip file so that it can be read.
No further operations will work after this method is called."""
if hasattr(self, 'bf') and self.bf:
self.bf.close()
self.bf = _ExceptionProxy()
@synchronize
def open(self, path, mode="r", **kwargs):
path = normpath(relpath(path))
if 'r' in mode:
if self.file_mode not in 'ra':
raise OperationFailedError("open file", path=path, msg="Big file must be opened for reading ('r') or appending ('a')")
try:
return self.entries[path].getfile(self.bf)
except KeyError:
raise ResourceNotFoundError(path)
if 'w' in mode:
raise OperationFailedError("open file", path=path, msg="Big file cannot be edited ATM")
raise ValueError("Mode must contain be 'r' or 'w'")
@synchronize
def getcontents(self, path):
if not self.exists(path):
raise ResourceNotFoundError(path)
path = normpath(path)
try:
contents = self.entries[path].getcontents(self.bf)
except KeyError:
raise ResourceNotFoundError(path)
except RuntimeError:
raise OperationFailedError("read file", path=path, msg="Big file must be oppened with 'r' or 'a' to read")
return contents
def desc(self, path):
if self.isdir(path):
return "Dir in big file: %s" % self.big_path
else:
return "File in big file: %s" % self.big_path
def isdir(self, path):
return self._path_fs.isdir(path)
def isfile(self, path):
return self._path_fs.isfile(path)
def exists(self, path):
return self._path_fs.exists(path)
@synchronize
def makedir(self, dirname, recursive=False, allow_recreate=False):
dirname = normpath(dirname)
if self.file_mode not in "wa":
raise OperationFailedError("create directory", path=dirname, msg="Big file must be opened for writing ('w') or appending ('a')")
if not dirname.endswith('/'):
dirname += '/'
self._add_resource(dirname)
def listdir(self, path="/", wildcard=None, full=False, absolute=False, dirs_only=False, files_only=False):
return self._path_fs.listdir(path, wildcard, full, absolute, dirs_only, files_only)
@synchronize
def getinfo(self, path):
if not self.exists(path):
raise ResourceNotFoundError(path)
path = normpath(path).lstrip('/')
info = {'size': 0}
if path in self.entries:
be = self.entries[path]
info['size'] = be.realSize
info['file_size'] = be.realSize
info['stored_size'] = be.storedSize
info['is_compressed'] = be.isCompressed
info['offset'] = be.offset
info['internal_filename'] = be.filename
info['filename'] = path
return info
def test_metadata_import_export(self):
"""Two checks:
- unknown metadata is preserved across import-export
- inherited metadata doesn't leak to children.
"""
system = self.get_system()
v = 'March 20 17:00'
url_name = 'test1'
start_xml = '''
<course org="{org}" course="{course}"
due="{due}" url_name="{url_name}" unicorn="purple">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html>
</chapter>
</course>'''.format(due=v, org=ORG, course=COURSE, url_name=url_name)
descriptor = system.process_xml(start_xml)
compute_inherited_metadata(descriptor)
# pylint: disable=W0212
print(descriptor, descriptor._field_data)
self.assertEqual(descriptor.due, ImportTestCase.date.from_json(v))
# Check that the child inherits due correctly
child = descriptor.get_children()[0]
self.assertEqual(child.due, ImportTestCase.date.from_json(v))
# need to convert v to canonical json b4 comparing
self.assertEqual(
ImportTestCase.date.to_json(ImportTestCase.date.from_json(v)),
child.xblock_kvs.inherited_settings['due'])
# Now export and check things
descriptor.runtime.export_fs = MemoryFS()
node = etree.Element('unknown')
descriptor.add_xml_to_node(node)
# Check that the exported xml is just a pointer
print("Exported xml:", etree.tostring(node))
self.assertTrue(is_pointer_tag(node))
# but it's a special case course pointer
self.assertEqual(node.attrib['course'], COURSE)
self.assertEqual(node.attrib['org'], ORG)
# Does the course still have unicorns?
with descriptor.runtime.export_fs.open(
'course/{url_name}.xml'.format(url_name=url_name)) as f:
course_xml = etree.fromstring(f.read())
self.assertEqual(course_xml.attrib['unicorn'], 'purple')
# the course and org tags should be _only_ in the pointer
self.assertTrue('course' not in course_xml.attrib)
self.assertTrue('org' not in course_xml.attrib)
# did we successfully strip the url_name from the definition contents?
self.assertTrue('url_name' not in course_xml.attrib)
# Does the chapter tag now have a due attribute?
# hardcoded path to child
with descriptor.runtime.export_fs.open('chapter/ch.xml') as f:
chapter_xml = etree.fromstring(f.read())
self.assertEqual(chapter_xml.tag, 'chapter')
self.assertFalse('due' in chapter_xml.attrib)
def test_tilejson(self):
with MemoryFS() as fs:
tj = json.loads(
Config('''{"metadata": {"id":"v1"}}''',
fs).tilejson("http://localhost/{id}/{z}/{x}/{y}.mvt"))
self.assertEqual(tj["tilejson"], "2.2.0")
self.assertEqual(tj["format"], "pbf")
self.assertEqual(tj["scheme"], "xyz")
self.assertEqual(tj["tiles"],
["http://localhost/v1/{z}/{x}/{y}.mvt"])
with MemoryFS() as fs:
fs.writetext("water.sql.jinja2", "select 1")
fs.writetext("ne-admin.sql.jinja2", "select 2")
fs.writetext("admin.sql.jinja2", "select 3")
fs.writetext("country.sql.jinja2", "select 4")
tj = json.loads(
Config(sample_config,
fs).tilejson("http://localhost/{id}/{z}/{x}/{y}.mvt"))
self.assertEqual(tj["name"], "name for tilejson, optional")
self.assertEqual(tj["description"],
"description for tilejson, optional")
self.assertEqual(tj["attribution"],
"attribution for tilejson, optional")
self.assertEqual(tj["version"], "version for tilejson, optional")
self.assertEqual(tj["bounds"],
[-180, -85.05112877980659, 180, 85.0511287798066])
self.assertEqual(tj["center"], [0, 0])
self.assertEqual(tj["minzoom"], 0)
self.assertEqual(tj["maxzoom"], 14)
# tilejsons have a list of layers you need to iterate through
water = {}
admin = {}
country_names = {}
self.assertEqual(len(tj["vector_layers"]), 3)
for l in tj["vector_layers"]:
if l["id"] == "water":
water = l
elif l["id"] == "admin":
admin = l
elif l["id"] == "country_names":
country_names = l
self.assertEqual(water["id"], "water")
self.assertEqual(water["description"], "Waterbody and ocean areas")
self.assertEqual(water["minzoom"], 0)
self.assertEqual(water["maxzoom"], 8)
self.assertEqual(water["geometry"], "unknown")
self.assertEqual(water["fields"], {"water": "Type of water"})
self.assertEqual(admin["id"], "admin")
self.assertEqual(admin["description"], "Administrative boundaries")
self.assertEqual(admin["minzoom"], 1)
self.assertEqual(admin["maxzoom"], 10)
self.assertEqual(admin["geometry"], "polygon")
self.assertEqual(admin["fields"],
{"admin_level": "Level of admin boundary"})
self.assertEqual(country_names["id"], "country_names")
self.assertEqual(country_names["description"],
"Points for country names")
self.assertEqual(country_names["minzoom"], 3)
self.assertEqual(country_names["maxzoom"], 14)
self.assertEqual(country_names["geometry"], "unknown")
self.assertEqual(country_names["fields"], {
"area": "Area of country",
"name": "Name of country"
})
def start_library(self):
console = self.console
from ...tools import get_moya_dir
from os.path import join, abspath
project_path = None
if self.args.location is not None:
library_path = self.args.location
else:
try:
project_path = get_moya_dir(self.args.project_location)
except:
console.error("Please run 'moya start library' inside your project directory, or specifiy the -o switch")
return False
library_path = abspath(join(project_path, './local/'))
cfg = None
if not self.args.location and project_path:
from ... import build
cfg = build.read_config(project_path, self.get_settings())
if not self.args.acceptdefaults:
console.table([[Cell("Moya Library Wizard", bold=True, fg="green", center=True)],
["""This will ask you a few questions, then create a new library in your Moya project based on your answers.
Default values are shown in grey (simply hit return to accept defaults). Some defaults may be taken from your ".bashrc" file, if it exists.
"""]])
author = self.get_author_details()
library = {}
library["title"] = LibraryTitle.ask(console, default=self.args.title)
longname = self.args.longname or make_name(author["organization"], library["title"])
longname = library["longname"] = LibraryLongName.ask(console, default=longname)
library["url"] = LibraryURL.ask(console, default="")
library["namespace"] = LibraryNamespace.ask(console, default="")
mount = None
appname = None
do_mount = DoMount.ask(console, default="yes")
if do_mount:
mount = Mount.ask(console, default=self.args.mount or "/{}/".format(make_name(library["title"])))
appname = AppName.ask(console, default=self.args.name or make_name(library["title"]))
data = dict(author=author,
library=library,
timezone=self.get_timezone())
actions = []
from ...command.sub import library_template
from fs.memoryfs import MemoryFS
from fs.opener import fsopendir
memfs = MemoryFS()
templatebuilder.compile_fs_template(memfs,
library_template.template,
data=data)
dest_fs = fsopendir(join(library_path, library["longname"]), create_dir=True, writeable=True)
continue_overwrite = 'overwrite'
if not dest_fs.isdirempty('.'):
if self.args.force:
continue_overwrite = 'overwrite'
elif self.args.new:
continue_overwrite = 'new'
else:
continue_overwrite = DirNotEmpty.ask(console, default="cancel")
if continue_overwrite != 'cancel':
if continue_overwrite == 'overwrite':
from fs.utils import copydir
copydir(memfs, dest_fs)
actions.append("Written library files to {}".format(dest_fs.getsyspath('.')))
elif continue_overwrite == 'new':
files_copied = copy_new(memfs, dest_fs)
table = [[
Cell("{} new file(s) written".format(len(files_copied)), fg="green", bold=True, center=True),
]]
for path in files_copied:
table.append([Cell(dest_fs.desc(path), bold=True, fg="black")])
console.table(table)
return 0
if cfg:
project_cfg = cfg['project']
location = project_cfg['location']
server_name = "main"
if location:
with fsopendir(project_path) as project_fs:
with project_fs.opendir(location) as server_fs:
from lxml.etree import fromstring, ElementTree, parse
from lxml.etree import XML, Comment
server_xml_path = server_fs.getsyspath(project_cfg['startup'])
root = parse(server_xml_path)
import_tag = XML('<import location="./local/{longname}" />\n\n'.format(**library))
import_tag.tail = "\n"
install_tag = None
if mount:
tag = '<install name="{appname}" lib="{longname}" mount="{mount}" />'
else:
tag = '<install name="{appname}" lib="{longname}" />'
install_tag = XML(tag.format(appname=appname,
longname=longname,
mount=mount))
install_tag.tail = "\n\n"
def has_child(node, tag, **attribs):
for el in node.findall(tag):
#items = dict(el.items())
if all(el.get(k, None) == v for k, v in attribs.items()):
return True
return False
for server in root.findall("{{http://moyaproject.com}}server[@docname='{}']".format(server_name)):
add_import_tag = not has_child(server, "{http://moyaproject.com}import", location="./local/{}".format(longname))
add_install_tag = not has_child(server, "{http://moyaproject.com}install", lib=longname) and install_tag is not None
if add_import_tag or add_install_tag:
comment = Comment("Added by 'moya start library'")
comment.tail = "\n"
server.append(comment)
if add_import_tag:
server.append(import_tag)
actions.append("Added <import> tag")
if add_install_tag:
server.append(install_tag)
actions.append("Added <install> tag")
if mount:
actions.append("Mounted application on {}".format(mount))
root.write(server_xml_path)
table = [[Cell("Library files written successfully!", fg="green", bold=True, center=True)]]
actions_text = "\n".join(" * " + action for action in actions)
table.append([Cell(actions_text, fg="blue", bold=True)])
table.append(["""A new library has been added to the project, containing some simple example functionality.\nSee http://moyaproject.com/docs/creatinglibraries/ for more information."""])
console.table(table)
return 0
console.text("No project files written.", fg="red", bold=True).nl()
return -1
class VirtualFilesystem(AbstractedFS):
"""Represents a virtual filesystem (currently only memory and s3 are supported)
"""
def __init__(self, root, cmd_channel):
AbstractedFS.__init__(self, root, cmd_channel)
self.cwd = root
self.type = cmd_channel.type
self.s3_bucket = cmd_channel.s3_bucket
self.aws_access_key = cmd_channel.aws_access_key
self.aws_secret_key = cmd_channel.aws_secret_key
self.seperator = cmd_channel.seperator
self.thread_synchronize = cmd_channel.thread_synchronize
self.key_sync_timeout = cmd_channel.key_sync_timeout
if not self.cmd_channel.fs_obj:
if self.type == "memory":
self.fs_obj = MemoryFS()
elif self.type == "s3":
self.fs_obj = S3FS(bucket=self.bucket, prefix=self.prefix, aws_access_key=self.aws_access_key, aws_secret_key=self.aws_secret_key, separator=self.seperator, thread_synchronize=self.thread_synchronize, key_sync_timeout=self.key_sync_timeout)
self.cmd_channel.fs_obj = self.fs_obj
else:
self.fs_obj = self.cmd_channel.fs_obj
def ftp2fs(self, ftppath):
return self.ftpnorm(ftppath)
def fs2ftp(self, fspath):
return fspath
def validpath(self, path):
# validpath was used to check symlinks escaping user home
# directory; this is no longer necessary.
return True
def open(self, filename, mode):
f = self.fs_obj.open(filename, mode)
f.name=filename
return f
def mkdir(self, path):
return self.fs_obj.makedir(path)
def chdir(self, path):
return self.fs_obj.opendir(path)
def listdir(self,path):
return self.fs_obj.listdir(path)
def rmdir(self, path):
return self.fs_obj.removedir(path)
def remove(self, path):
return self.fs_obj.remove(path)
def rename(self, src, dst):
return self.fs_obj.rename(src, dst)
def chmod(self, path, mode):
return True
def readlink(self, path):
return self.ftp2fs(path)
def isfile(self, path):
return self.fs_obj.isfile(path)
def islink(self, path):
return False
def getsize(self, path):
return self.fs_obj.getsize(path)
def getmtime(self, path):
return self.fs_obj.getinfo(path)['modified_time']
def realpath(self, path):
return path
def lexists(self, path):
return self.fs_obj.exists(path)
def mkstemp(self, suffix='', prefix='', mode='wb'):
from tempfile import _RandomNameSequence as RandomName
name = RandomName()
if suffix != '':
suffix = 'tmp'
fname = suffix + name.next()
return self.fs_obj.open(fname,mode)
def parted_file(self):
fs = MemoryFS()
mode = "wb+"
path = "cuckoo.tar"
parts = [FilePart(fs.open("cuckoo.tar.part0", mode)), (fs.open("cuckoo.tar.part1", mode))]
return PartedFile(path=path, mode=mode, fs=fs, max_part_size=kb(4), parts=parts)
def start_project(self):
console = self.console
if not self.args.acceptdefaults:
console.table([[Cell("Moya Project Wizard", bold=True, fg="green", center=True)],
["""This will ask you a few questions, then create a new Moya project based on your answers.
Default values are shown in blue (hit return to accept defaults). Some defaults may be taken from your ".moyarc" file, if it exists."""]])
author = self.get_author_details()
project = {}
project["title"] = ProjectTitle.ask(console, default=self.args.title)
longname = make_name(author["organization"], project["title"])
project["database"] = Database.ask(console, default='y')
if project["database"]:
project["auth"] = Auth.ask(console, default='y')
project['signup'] = Signup.ask(console, default='y')
project["pages"] = Pages.ask(console, default='y')
project["feedback"] = Feedback.ask(console, default='y')
project["blog"] = Blog.ask(console, default='y')
project["comments"] = project.get("feedback", False) or project.get("pages", False)
project["wysihtml5"] = project.get("feedback", False) or project.get("pages", False)
project['jsonrpc'] = JSONRPC.ask(console, default='y')
dirname = longname.split('.', 1)[-1].replace('.', '_')
dirname = ProjectDirName.ask(console, default="./" + dirname)
data = dict(author=author,
project=project,
timezone=self.get_timezone())
from ...command.sub import project_template
from fs.memoryfs import MemoryFS
from fs.opener import fsopendir
memfs = MemoryFS()
templatebuilder.compile_fs_template(memfs,
project_template.template,
data=data)
dest_fs = fsopendir(self.args.location or dirname, create_dir=True, writeable=True)
continue_overwrite = 'overwrite'
if not dest_fs.isdirempty('.'):
if self.args.force:
continue_overwrite = 'overwrite'
elif self.args.new:
continue_overwrite = 'new'
else:
continue_overwrite = DirNotEmpty.ask(console, default="cancel")
if continue_overwrite == 'overwrite':
from fs.utils import copydir
copydir(memfs, dest_fs)
console.table([[Cell("Project files written successfully!", fg="green", bold=True, center=True)],
["""See readme.txt in the project directory for the next steps.\n\nBrowse to http://moyaproject.com/gettingstarted/ if you need further help."""]])
return 0
elif continue_overwrite == 'new':
files_copied = copy_new(memfs, dest_fs)
table = [[
Cell("{} new file(s) written".format(len(files_copied)), fg="green", bold=True, center=True),
]]
for path in files_copied:
table.append([Cell(dest_fs.desc(path), bold=True, fg="black")])
console.table(table)
return 0
console.text("No project files written.", fg="red", bold=True).nl()
return -1
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir):
from fs.memoryfs import MemoryFS
memfs = MemoryFS()
if create_dir:
memfs = memfs.makeopendir(fs_path)
return memfs, None
def test_metadata_import_export(self):
"""Two checks:
- unknown metadata is preserved across import-export
- inherited metadata doesn't leak to children.
"""
system = self.get_system()
v = "March 20 17:00"
url_name = "test1"
start_xml = """
<course org="{org}" course="{course}"
due="{due}" url_name="{url_name}" unicorn="purple">
<chapter url="hi" url_name="ch" display_name="CH">
<html url_name="h" display_name="H">Two houses, ...</html>
</chapter>
</course>""".format(
due=v, org=ORG, course=COURSE, url_name=url_name
)
descriptor = system.process_xml(start_xml)
compute_inherited_metadata(descriptor)
# pylint: disable=W0212
print(descriptor, descriptor._field_data)
self.assertEqual(descriptor.due, ImportTestCase.date.from_json(v))
# Check that the child inherits due correctly
child = descriptor.get_children()[0]
self.assertEqual(child.due, ImportTestCase.date.from_json(v))
# need to convert v to canonical json b4 comparing
self.assertEqual(
ImportTestCase.date.to_json(ImportTestCase.date.from_json(v)), child.xblock_kvs.inherited_settings["due"]
)
# Now export and check things
resource_fs = MemoryFS()
exported_xml = descriptor.export_to_xml(resource_fs)
# Check that the exported xml is just a pointer
print("Exported xml:", exported_xml)
pointer = etree.fromstring(exported_xml)
self.assertTrue(is_pointer_tag(pointer))
# but it's a special case course pointer
self.assertEqual(pointer.attrib["course"], COURSE)
self.assertEqual(pointer.attrib["org"], ORG)
# Does the course still have unicorns?
with resource_fs.open("course/{url_name}.xml".format(url_name=url_name)) as f:
course_xml = etree.fromstring(f.read())
self.assertEqual(course_xml.attrib["unicorn"], "purple")
# the course and org tags should be _only_ in the pointer
self.assertTrue("course" not in course_xml.attrib)
self.assertTrue("org" not in course_xml.attrib)
# did we successfully strip the url_name from the definition contents?
self.assertTrue("url_name" not in course_xml.attrib)
# Does the chapter tag now have a due attribute?
# hardcoded path to child
with resource_fs.open("chapter/ch.xml") as f:
chapter_xml = etree.fromstring(f.read())
self.assertEqual(chapter_xml.tag, "chapter")
self.assertFalse("due" in chapter_xml.attrib)
def setUp(self):
self.fs = MemoryFS()
self.cache = cache.filecache.FileCache("test", "ns1", fs=self.fs)
self.cache2 = cache.filecache.FileCache("test", "ns2", fs=self.fs)
def setUp(self):
super(SplitTestModuleTest, self).setUp()
self.course_id = 'test_org/test_course_number/test_run'
# construct module
course = xml.CourseFactory.build()
sequence = xml.SequenceFactory.build(parent=course)
split_test = SplitTestModuleFactory(
parent=sequence,
attribs={
'user_partition_id':
'0',
'group_id_to_child':
'{"0": "i4x://edX/xml_test_course/html/split_test_cond0", "1": "i4x://edX/xml_test_course/html/split_test_cond1"}' # pylint: disable=line-too-long
})
xml.HtmlFactory(parent=split_test,
url_name='split_test_cond0',
text='HTML FOR GROUP 0')
xml.HtmlFactory(parent=split_test,
url_name='split_test_cond1',
text='HTML FOR GROUP 1')
self.course = self.process_xml(course)
self.course_sequence = self.course.get_children()[0]
self.module_system = get_test_system()
self.module_system.descriptor_runtime = self.course._runtime # pylint: disable=protected-access
self.course.runtime.export_fs = MemoryFS()
# Create mock partition service, as these tests are running with XML in-memory system.
self.course.user_partitions = [
self.user_partition,
UserPartition(
MINIMUM_STATIC_PARTITION_ID, 'second_partition',
'Second Partition', [
Group(unicode(MINIMUM_STATIC_PARTITION_ID + 1), 'abel'),
Group(unicode(MINIMUM_STATIC_PARTITION_ID + 2), 'baker'),
Group("103", 'charlie')
], MockUserPartitionScheme())
]
partitions_service = MockPartitionService(
self.course,
course_id=self.course.id,
track_function=Mock(name='track_function'),
)
self.module_system._services['partitions'] = partitions_service # pylint: disable=protected-access
# Mock user_service user
user_service = Mock()
user = Mock(username='******',
email='*****@*****.**',
is_staff=False,
is_active=True)
user_service._django_user = user
self.module_system._services['user'] = user_service # pylint: disable=protected-access
self.split_test_module = self.course_sequence.get_children()[0]
self.split_test_module.bind_for_student(self.module_system, user.id)
# Create mock modulestore for getting the course. Needed for rendering the HTML
# view, since mock services exist and the rendering code will not short-circuit.
mocked_modulestore = Mock()
mocked_modulestore.get_course.return_value = self.course
self.split_test_module.system.modulestore = mocked_modulestore
def test_update_from_single_version(self) -> None:
fs = MemoryFS()
mv = Multiversioned(fs)
d = {
"file1.txt": "file1",
"file2.txt": "file2",
"dir1": {
"file1.txt": "file1",
"file2.txt": "file2"
},
"dir2": {
"file1.txt": "file1",
"file2.txt": "file2"
},
}
bundle_lidvid = LIDVID("urn:nasa:pds:b::1.0")
bundle_lid = bundle_lidvid.lid()
no_lidvids: Set[LIDVID] = set()
def create_bundle() -> None:
lidvids = [create_collection(bundle_lid, c) for c in ["c1", "c2"]]
contents = dictionary_to_contents(set(lidvids), d)
mv.add_contents_if(is_new, bundle_lid, contents)
def create_collection(bundle_lid: LID, c: str) -> LIDVID:
lid = bundle_lid.extend_lid(c)
lidvids = [create_product(lid, p) for p in ["p1", "p2"]]
contents = dictionary_to_contents(set(lidvids), d)
return mv.add_contents_if(is_new, lid, contents)
def create_product(coll_lid: LID, p: str) -> LIDVID:
lid = coll_lid.extend_lid(p)
contents = dictionary_to_contents(no_lidvids, d)
return mv.add_contents_if(is_new, lid, contents)
create_bundle()
vv = VersionView(mv, bundle_lidvid)
c = COWFS(vv, MemoryFS(), MemoryFS())
path = "/b$/c2$/p1$/dir1/file2.txt"
c.writetext(path, "xxxx")
latest_lidvid = mv.latest_lidvid(LID("urn:nasa:pds:b"))
# Update from the COWFS.
mv.update_from_single_version(is_new, c)
self.assertNotEqual(latest_lidvid,
mv.latest_lidvid(LID("urn:nasa:pds:b")))
latest_lidvid = mv.latest_lidvid(LID("urn:nasa:pds:b"))
# changed files are changed
self.assertEqual("file2", fs.readtext("b/c2/p1/v$1.0/dir1/file2.txt"))
self.assertEqual("xxxx", fs.readtext("b/c2/p1/v$2.0/dir1/file2.txt"))
# unchanged files are unchanged
self.assertEqual("file1", fs.readtext("b/c2/p1/v$1.0/dir1/file1.txt"))
self.assertEqual("file1", fs.readtext("b/c2/p1/v$2.0/dir1/file1.txt"))
# Change started in b/c2/p1. Check which versions are affected.
self.assertEqual(
VID("2.0"),
cast(LIDVID, mv.latest_lidvid(LID("urn:nasa:pds:b"))).vid())
self.assertEqual(
VID("1.0"),
cast(LIDVID, mv.latest_lidvid(LID("urn:nasa:pds:b:c1"))).vid())
self.assertEqual(
VID("2.0"),
cast(LIDVID, mv.latest_lidvid(LID("urn:nasa:pds:b:c2"))).vid())
self.assertEqual(
VID("2.0"),
cast(LIDVID, mv.latest_lidvid(LID("urn:nasa:pds:b:c2:p1"))).vid(),
)
self.assertEqual(
VID("1.0"),
cast(LIDVID, mv.latest_lidvid(LID("urn:nasa:pds:b:c2:p2"))).vid(),
)
# Now try updating again. Nothing should change.
mv.update_from_single_version(is_new, c)
self.assertEqual(latest_lidvid,
mv.latest_lidvid(LID("urn:nasa:pds:b")))
