def test_listdir_duplicates(self):
m1 = MemoryFS()
m2 = MemoryFS()
m1.touch("foo")
m2.touch("foo")
multi_fs = MultiFS()
multi_fs.add_fs("m1", m1)
multi_fs.add_fs("m2", m2)
self.assertEqual(multi_fs.listdir("/"), ["foo"])
def test_listdir_duplicates(self):
m1 = MemoryFS()
m2 = MemoryFS()
m1.touch('foo')
m2.touch('foo')
multi_fs = MultiFS()
multi_fs.add_fs('m1', m1)
multi_fs.add_fs('m2', m2)
self.assertEqual(multi_fs.listdir(u'/'), ['foo'])
def __init__(self, project_fs=None, breakpoint=False, strict=False, test_build=False, develop=False):
self.project_fs = project_fs
self.strict = strict
self.test_build = test_build
self.develop = develop
self.registry = ElementRegistry()
self.libs = {}
self.apps = OrderedDict()
self.apps_by_lib = defaultdict(list)
self.app_settings = defaultdict(SettingsContainer)
self.app_system_settings = defaultdict(SettingsContainer)
self.cfg = None
self.settings = SettingsContainer()
self.templates_fs = MultiFS()
self.data_fs = MultiFS()
self.filesystems = FSContainer({'templates': self.templates_fs,
'data': self.data_fs})
self.filters = FilterContainer(self)
self.template_engines = {}
self.database_engines = {}
self.caches = {}
self.mail_servers = {}
self.default_mail_server = None
self.default_db_engine = None
self.debug = False
self.struct = False
self.auto_reload = False
self.known_namespaces = set()
self.sites = Sites()
self.breakpoint = breakpoint
self.suppress_breakpoints = False
self.data_tags = defaultdict(list)
self.data_tags_by_lib = defaultdict(lambda: defaultdict(list)) # awesome
self.preflight = False
self.log_signals = False
self.debug_echo = False
self.debug_memory = False
self.lib_paths = None
self._lib_database = None
self.log_logger = None
self.log_color = True
self.log_width = None
self.media_urls = None
self.media_app = None
self.failed_documents = []
self.enum = {}
self.enum_by_lib = {}
self.signals = Signals()
self._moyarc = None
self.console = self.create_console()
def test_opener(self):
"""Test use of FS URLs."""
multi_fs = MultiFS()
with self.assertRaises(TypeError):
multi_fs.add_fs(u'foo', 5)
multi_fs.add_fs(u'f1', u'mem://')
multi_fs.add_fs(u'f2', u'temp://')
self.assertIsInstance(multi_fs.get_fs(u'f1'), MemoryFS)
def test_opener(self):
"""Test use of FS URLs."""
multi_fs = MultiFS()
with self.assertRaises(TypeError):
multi_fs.add_fs("foo", 5)
multi_fs.add_fs("f1", "mem://")
multi_fs.add_fs("f2", "temp://")
self.assertIsInstance(multi_fs.get_fs("f1"), MemoryFS)
def test_auto_close(self):
"""Test MultiFS auto close is working"""
multi_fs = MultiFS()
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.addfs('m1', m1)
multi_fs.addfs('m2', m2)
self.assert_(not m1.closed)
self.assert_(not m2.closed)
multi_fs.close()
self.assert_(m1.closed)
self.assert_(m2.closed)
def test_no_auto_close(self):
"""Test MultiFS auto close can be disables"""
multi_fs = MultiFS(auto_close=False)
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.addfs('m1', m1)
multi_fs.addfs('m2', m2)
self.assert_(not m1.closed)
self.assert_(not m2.closed)
multi_fs.close()
self.assert_(not m1.closed)
self.assert_(not m2.closed)
def _find_extensions(paths):
'''Iterate the paths, in order, finding extensions and adding them to
the return dict.'''
extension_kinds = ['check', 'configure', 'write']
efs = MultiFS()
map(lambda x: efs.addfs(x, OSFS(x)), paths)
def get_extensions(kind):
return {os.path.splitext(x)[0]: efs.getsyspath(x)
for x in efs.walkfiles('.', '*.%s' % kind)}
return {e: get_extensions(e) for e in extension_kinds}
def _find_extensions(paths):
'''Iterate the paths, in order, finding extensions and adding them to
the return dict.'''
extension_kinds = ['check', 'configure', 'write']
efs = MultiFS()
map(lambda x: efs.addfs(x, OSFS(x)), paths)
def get_extensions(kind):
return {os.path.splitext(x)[0]: efs.getsyspath(x)
for x in efs.walkfiles('.', '*.%s' % kind)}
return {e: get_extensions(e) for e in extension_kinds}
class _AnsiblePlaybookSources(object):
def __init__(self, **kwargs):
self.fs = MultiFS()
self.config = kwargs['config']
self.basepath = kwargs['basepath']
def add_folder(self, folder_path, dest=""):
real_path = search_path(folder_path, self.basepath, getcwd())
if real_path is None:
raise RuntimeError("Couldn't find ansible playbook sources at %s" % folder_path)
self.fs.addfs(dest, OSFS(real_path))
# TODO: In the future...
# def add_git(self, **kwargs):
# def add_file_url(self, url):
# Plain method to embed the playbook contents into the cloudformation template
def _fs_to_cfninit_plain(self):
from base64 import b64encode
# Create a metadata object that, when executed by cfn-init, will result in the files
# being created in the instance
from os.path import join
files_obj = {}
target = self.config['_inst_playbook_path']
for (fs_dir, fs_dirfiles) in self.fs.walk():
# TODO: more deterministic order of walking through the files
if fs_dir[0] == '/': fs_dir = fs_dir[1:]
t = join(target, fs_dir)
for f in fs_dirfiles:
src = join(fs_dir, f)
dest = join(t, f)
encoding= None
try:
contents= self.fs.getcontents(src, mode='rb')
contents.decode('utf-8')
except UnicodeDecodeError:
encoding= "base64"
contents= b64encode(contents)
files_obj[dest] = dict(
content= contents,
owner= self.config['inst_user'],
group= self.config['inst_group'],
mode= "000640"
)
files_obj[dest].update( {"encoding": encoding} if encoding else {} )
return files_obj
def deploy(self, iscm):
# Create cfn-init stanzas for creating the added files
iscm.iscm_cfninit_add_config({ "files": self._fs_to_cfninit_plain() }, "_ansible_playbooks_install")
def __init__(self, project_fs=None, breakpoint=False):
self.project_fs = project_fs
self.registry = ElementRegistry()
self.libs = {}
self.apps = OrderedDict()
self.apps_by_lib = defaultdict(list)
self.app_settings = defaultdict(SettingsContainer)
self.app_system_settings = defaultdict(SettingsContainer)
self.cfg = None
self.settings = SettingsContainer()
self.templates_fs = MultiFS()
self.data_fs = MultiFS()
self.filesystems = FSContainer({'templates': self.templates_fs,
'data': self.data_fs})
self.filters = FilterContainer(self)
self.template_engines = {}
self.database_engines = {}
self.caches = {}
self.mail_servers = {}
self.default_mail_server = None
self.default_db_engine = None
self.debug = False
self.auto_reload = False
self.known_namespaces = set()
self.sites = Sites()
self.breakpoint = breakpoint
self.suppress_breakpoints = False
self.data_tags = defaultdict(list)
self.data_tags_by_lib = defaultdict(lambda: defaultdict(list)) # awesome
self.preflight = False
self.log_signals = False
self.debug_echo = False
self.log_logger = None
self.log_color = True
self.log_width = None
self.media_urls = None
self.media_app = None
self.failed_documents = []
self.enum = {}
self.enum_by_lib = {}
self.signals = Signals()
self._moyarc = None
self.console = self.create_console()
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir):
from fs.multifs import MultiFS
from ConfigParser import ConfigParser
cfg = ConfigParser()
if '#' in fs_path:
path, section = fs_path.split('#', 1)
else:
path = fs_path
section = 'fs'
cfg.readfp(registry.open(path))
multi_fs = MultiFS()
for name, fs_url in cfg.items(section):
multi_fs.addfs(name, registry.opendir(fs_url, create_dir=create_dir))
return multi_fs, ''
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir):
from fs.multifs import MultiFS
from configparser import ConfigParser
cfg = ConfigParser()
if '#' in fs_path:
path, section = fs_path.split('#', 1)
else:
path = fs_path
section = 'fs'
cfg.readfp(registry.open(path))
multi_fs = MultiFS()
for name, fs_url in cfg.items(section):
multi_fs.addfs(name, registry.opendir(fs_url, create_dir=create_dir))
return multi_fs, ''
def get_multi_fs(root):
"""Create filesystem combining the server (if connected) with profile and scenario
containers in blob storage. The priority is in descending order, so the server will
be used first if possible
:param str root: root directory on server
:return: (*fs.base.FS*) -- filesystem instance
"""
scenario_data = get_blob_fs("scenariodata")
profiles = get_blob_fs("profiles")
mfs = MultiFS()
try:
ssh_fs = get_ssh_fs(root)
mfs.add_fs("ssh_fs", ssh_fs, write=True, priority=3)
except: # noqa
print("Could not connect to ssh server")
mfs.add_fs("profile_fs", profiles, priority=2)
mfs.add_fs("scenario_fs", scenario_data, priority=1)
remotes = ",".join([f[0] for f in mfs.iterate_fs()])
print(f"Initialized remote filesystem with {remotes}")
return mfs
def test_no_auto_close(self):
"""Test MultiFS auto close can be disables"""
multi_fs = MultiFS(auto_close=False)
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.addfs('m1', m1)
multi_fs.addfs('m2', m2)
self.assert_(not m1.closed)
self.assert_(not m2.closed)
multi_fs.close()
self.assert_(not m1.closed)
self.assert_(not m2.closed)
def test_auto_close(self):
"""Test MultiFS auto close is working"""
multi_fs = MultiFS()
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.addfs('m1', m1)
multi_fs.addfs('m2', m2)
self.assert_(not m1.closed)
self.assert_(not m2.closed)
multi_fs.close()
self.assert_(m1.closed)
self.assert_(m2.closed)
def test_auto_close(self):
"""Test MultiFS auto close is working"""
multi_fs = MultiFS()
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.add_fs("m1", m1)
multi_fs.add_fs("m2", m2)
self.assertFalse(m1.isclosed())
self.assertFalse(m2.isclosed())
multi_fs.close()
self.assertTrue(m1.isclosed())
self.assertTrue(m2.isclosed())
def test_no_auto_close(self):
"""Test MultiFS auto close can be disabled"""
multi_fs = MultiFS(auto_close=False)
self.assertEqual(repr(multi_fs), "MultiFS(auto_close=False)")
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.add_fs("m1", m1)
multi_fs.add_fs("m2", m2)
self.assertFalse(m1.isclosed())
self.assertFalse(m2.isclosed())
multi_fs.close()
self.assertFalse(m1.isclosed())
self.assertFalse(m2.isclosed())
def test_multiple_fs_with_use_syspath(self, ctx):
testfs = ctx << fs.open_fs('mem://')
self.build_fs(testfs, ctx)
self.build_zipfs()
multi_fs = MultiFS()
multi_fs.add_fs('memory', testfs)
multi_fs.add_fs('zip', fs.open_fs("zip://test.zip"))
env = self.build_env(multi_fs, use_syspath=True)
source, path, _ = env.loader.get_source(None, "template_in_zip.j2")
self.assertEqual(path, "template_in_zip.j2")
os.unlink("test.zip")
def get_scenario_fs():
"""Create filesystem combining the server (if connected) with blob storage,
prioritizing the server if connected.
:return: (*fs.base.FS*) -- filesystem instance
"""
scenario_data = get_blob_fs("scenariodata")
mfs = MultiFS()
try:
ssh_fs = get_ssh_fs(server_setup.DATA_ROOT_DIR)
mfs.add_fs("ssh_fs", ssh_fs, write=True, priority=2)
except: # noqa
print("Could not connect to ssh server")
mfs.add_fs("scenario_fs", scenario_data, priority=1)
remotes = ",".join([f[0] for f in mfs.iterate_fs()])
print(f"Initialized remote filesystem with {remotes}")
return mfs
def test_multiple_fs(self, ctx):
testfs = ctx << fs.open_fs('mem://')
self.build_fs(testfs, ctx)
self.build_zipfs()
multi_fs = MultiFS()
multi_fs.add_fs('memory', testfs)
multi_fs.add_fs('zip', fs.open_fs("zip://test.zip"))
env = self.build_env(multi_fs)
template = env.get_template("dir/nested.j2")
self.assertEqual(template.render(),
"<html>this is a nested template !</html>")
template = env.get_template("template_in_zip.j2")
self.assertEqual(template.render(),
"<html>this template is in a zip</html>")
self.assertRaises(jinja2.TemplateNotFound, env.get_template,
"other.j2")
source, path, _ = env.loader.get_source(None, "template_in_zip.j2")
self.assertEqual(path, "template_in_zip.j2")
class Archive(object):
_re_element_ref_match = re.compile(
r"^(.+\..+)#(.*)$|^(.+)#(.*)$|^#(.+)$", re.UNICODE
).match
def __init__(
self,
project_fs=None,
breakpoint=False,
strict=False,
test_build=False,
develop=False,
):
self.project_fs = project_fs
self.strict = strict
self.test_build = test_build
self.develop = develop
self.registry = ElementRegistry()
self.libs = {}
self.apps = OrderedDict()
self.apps_by_lib = defaultdict(list)
self.app_settings = defaultdict(SettingsContainer)
self.app_system_settings = defaultdict(SettingsContainer)
self.cfg = None
self.settings = SettingsContainer()
self.templates_fs = MultiFS()
self.data_fs = MultiFS()
self.filesystems = FSContainer(
{"templates": self.templates_fs, "data": self.data_fs}
)
self.filters = FilterContainer(self)
self.template_engines = {}
self.database_engines = {}
self.caches = {}
self.mail_servers = {}
self.default_mail_server = None
self.default_db_engine = None
self.debug = False
self.struct = False
self.auto_reload = False
self.known_namespaces = set()
self.sites = Sites()
self.breakpoint = breakpoint
self.suppress_breakpoints = False
self.data_tags = defaultdict(list)
self.data_tags_by_lib = defaultdict(lambda: defaultdict(list)) # awesome
self.preflight = False
self.log_signals = False
self.debug_echo = False
self.debug_memory = False
self.lib_paths = None
self._lib_database = None
self.log_logger = None
self.log_color = True
self.log_width = None
self.media_urls = None
self.media_app = None
self.failed_documents = []
self.enum = {}
self.enum_by_lib = {}
self.signals = Signals()
self._moyarc = None
self.console = self.create_console()
def __repr__(self):
return "<archive>"
@property
def media_url(self):
return self.media_urls[0] if self.media_urls else None
@property
def moyarc(self):
if self._moyarc is None:
try:
with io.open(os.path.expanduser("~/.moyarc"), "rt") as f:
self._moyarc = settings.SettingsContainer.read_from_file(f)
except IOError:
self._moyarc = settings.SettingsContainer()
return self._moyarc
@property
def lib_database(self):
if self._lib_database is None:
start = time()
self._lib_database = self.scan_libs(self.lib_paths, self.project_fs)
log.debug(
"%s scanned %i libs %0.1fms",
self,
len(self._lib_database),
(time() - start) * 1000.0,
)
return self._lib_database
def find_lib(self, version_spec):
options = []
_version_spec = versioning.VersionSpec(version_spec)
for name, version, path, dir_name in self.lib_database:
if _version_spec.name != name:
continue
if _version_spec.comparisons and not _version_spec.compare(version):
continue
if "://" in path:
base_fs = open_fs(path)
else:
base_fs = self.project_fs.opendir(path)
lib_fs = base_fs.opendir(dir_name)
return lib_fs
def get_relative_path(self, path):
"""Get a relative path from the project base"""
base = self.project_fs.getsyspath("/", allow_none=True)
if base is None:
return path
return relativefrom(base, path)
def open_fs(self, fs_url, create=False):
if isinstance(fs_url, text_type):
if "://" in fs_url:
fs = open_fs(fs_url, create=create)
else:
if create:
self.project_fs.makedirs(fs_url, recreate=True)
fs = self.project_fs.opendir(fs_url)
else:
fs = self.project_fs.opendir(fs_url)
else:
fs = fs_url
return fs
def get_console_file(self):
if self.log_logger:
console_file = logtools.LoggerFile(self.log_logger)
else:
console_file = None
return console_file
def create_console(self):
console = Console(
out=self.get_console_file(),
nocolors=not (
self.log_color and self.moyarc.get_bool("console", "color", True)
),
width=self.log_width or None,
)
return console
@classmethod
def scan_libs(cls, lib_paths, base_fs):
"""Read libs from paths."""
libs = []
for path in lib_paths:
try:
if "://" in path:
libs_fs = open_fs(path)
else:
libs_fs = base_fs.opendir(path)
except FSError as error:
startup_log.warning("unable to read from '%s' (%s)", path, error)
continue
for resource in libs_fs.filterdir("/", exclude_files=["*"]):
with libs_fs.opendir(resource.name) as lib_fs:
try:
with lib_fs.open("lib.ini", "rb") as ini_file:
lib_settings = SettingsContainer.read_from_file(ini_file)
except ResourceNotFound:
continue
name = lib_settings.get("lib", "name", None)
if name is None:
continue
version = lib_settings.get("lib", "version", None)
if version is None:
continue
libs.append((name, version, path, resource.name))
return libs
def build_libs(self, ignore_errors=False):
if self.test_build:
ignore_errors = True
libs = [lib for lib in itervalues(self.libs) if not lib.built]
if not libs:
return
start = time()
self.build([doc for lib in libs for doc in lib.documents], log_time=False)
for lib in libs:
lib.finalize(ignore_errors=ignore_errors)
startup_log.debug("%s built libraries %0.1fms", self, (time() - start) * 1000.0)
def build(self, documents, context=None, log_time=True, fs=None):
"""Build all documents in the library"""
# This handles tags defined out of order
if fs is not None:
self.project_fs = fs
start = time()
if isinstance(documents, Document):
documents = [documents]
else:
documents = list(documents)
build_queue = deque()
for doc in documents:
if not doc.document_finalized and doc.structure:
build_queue.append((None, [doc.structure.root_node]))
unbuildable = set()
unbuildable_clear = unbuildable.clear
unbuilt = []
if context is None:
context = Context()
context_root = context.root
get_doc_id = attrgetter("doc_id")
while build_queue:
parent_element, nodes = build_queue[0]
if nodes:
node = nodes.pop()
context_root["_lib_long_name"] = node.lib_long_name
element = node.build(self, context)
if element:
unbuildable_clear()
build_queue.appendleft(
(element, sorted(node.children, key=get_doc_id, reverse=True))
)
else:
if element is not None:
if node in unbuildable:
# A previously deferred node, but we still can't built it
build_queue.popleft() # Can't process siblings either
unbuilt.append(node)
else:
# Defer this node till later
nodes.append(node)
build_queue.rotate(-1)
unbuildable.add(node)
else:
if parent_element is not None:
unbuildable_clear()
try:
parent_element.finalize(context)
except Exception as e:
raise
failed_doc = FailedDocument(
path=node.structure.document.location,
code=node.structure.xml,
line=node.source_line,
col=None,
msg=text_type(e),
)
self.failed_documents.append(failed_doc)
build_queue.popleft()
if unbuilt:
for node in unbuilt:
nearest = nearest_word(
node.tag_name, self.registry.get_elements_in_xmlns(node.xmlns)
)
msg = "unknown tag {} in {}".format(
node.tag_display_name, node.structure
)
diagnosis = None
if nearest:
diagnosis = "did you mean <{}>?".format(nearest)
else:
find_xmlns = self.registry.find_xmlns(node.tag_name)
if find_xmlns:
diagnosis = "did you mean <{}> in XMLNS '{}'?".format(
node.tag_name, find_xmlns
)
failed_doc = FailedDocument(
path=node.structure.document.location,
code=node.structure.xml,
line=node.source_line,
col=None,
msg=msg,
diagnosis=diagnosis,
)
self.failed_documents.append(failed_doc)
return False
if self.strict:
failed = 0
for doc in documents:
failed += self.check_attributes(doc)
if failed:
startup_log.debug("%s %s strict check(s) failed", self, failed)
else:
startup_log.debug("%s strict checks passed", self)
for doc in documents:
doc.document_finalize(context)
if log_time:
doc_text = ", ".join(text_type(doc) for doc in documents)
startup_log.debug("%s built %0.1fms", doc_text, (time() - start) * 1000.0)
return True
def check_attributes(self, doc):
failed = 0
for element_name, element in doc.elements.items():
for k, attribute in element._tag_attributes.items():
if k in element._attrs:
attr_text = element._attrs[k]
error = attribute.type.check(attr_text)
if error:
failed += 1
failed_doc = FailedDocument(
path=doc.location,
code=element._code,
line=element.source_line or 0,
col=None,
msg="error in parameter '{}'; {}".format(k, error),
diagnosis="This check is performed when [project]/strict is enabled, or with 'moya runserver --strict' switch",
)
self.failed_documents.append(failed_doc)
return failed
def populate_context(self, context):
from .context.expressiontime import ExpressionDateTime
root = context.root
root["libs"] = self.libs
root["apps"] = self.apps
root["filters"] = self.filters
root["debug"] = self.debug
root["develop"] = self.develop
root["fs"] = self.get_context_filesystems()
root["now"] = ExpressionDateTime.utcnow()
from . import __version__
root["moya"] = {"version": __version__}
root["enum"] = self.enum
root["media_url"] = self.media_url
root["secret"] = self.secret
context.set_dynamic(
".app", lambda context: getattr(context.get(".call", None), "app")
)
@classmethod
def get_callable_from_document(
cls,
path,
element_ref,
breakpoint=False,
fs="./",
default_context=False,
archive=None,
lib=None,
):
"""Shortcut that imports a single document and returns a callable"""
if archive is None:
archive = cls()
if lib is None:
lib = archive.create_library(long_name="moya.run", namespace=namespaces.run)
lib.import_document(fs, path)
archive.build_libs()
app, element = lib.documents[0].get_element(element_ref)
if element is None:
raise errors.ElementNotFoundError(element_ref, app=app)
call = CallableElement(archive, element, app, breakpoint=breakpoint)
if default_context:
def do_call(*args, **kwargs):
c = Context()
c["console"] = Console()
return call(c, *args, **kwargs)
do_call.archive = archive
return do_call
return call
@classmethod
def parse_element_ref(cls, s, cache={}):
try:
return cache[s]
except KeyError:
match = cls._re_element_ref_match(s)
if match is None:
result = None, None, s
else:
libname, lib_elementname, appname, app_elementname, docname = (
match.groups()
)
result = appname, libname, app_elementname or lib_elementname or docname
cache[s] = result
return result
def get_library(self, library_name):
"""Get a library from either its short name, or its long name"""
try:
return self.libs[library_name]
except KeyError:
raise errors.UnknownLibraryError(lib=library_name)
def has_library(self, library_name):
return library_name in self.libs
def load_library_from_module(self, py, **kwargs):
__import__(py)
module = sys.modules[py]
location = os.path.dirname(os.path.abspath(module.__file__))
import_fs = open_fs(location)
lib = self.load_library(import_fs, **kwargs)
return lib
def load_library(
self,
import_fs,
priority=None,
template_priority=None,
data_priority=None,
long_name=None,
rebuild=False,
):
"""Load a new library in to the archive"""
lib = self.create_library(import_fs, long_name=long_name, rebuild=rebuild)
if priority is not None:
lib.priority = priority
lib.data_priority = data_priority
if lib.templates_info:
fs_url = lib.templates_info["location"]
if template_priority is None:
try:
template_priority = int(lib.templates_info.get("priority", "0"))
except ValueError:
startup_log.error(
"{} invalid value for [templates]/priority, assuming 0".format(
lib
)
)
template_priority = 0
lib.template_priority = template_priority
if "://" in fs_url:
fs = open_fs(fs_url)
else:
fs = import_fs.opendir(fs_url)
self.templates_fs.add_fs(lib.long_name, fs, priority=template_priority)
return lib
def get_or_create_library(self, long_name, import_fs=None):
"""Get a library, or create it if it doesn't exists"""
if long_name is not None and long_name in self.libs:
return self.libs[long_name]
return self.create_library(import_fs)
def create_library(
self, import_fs=None, long_name=None, namespace=None, rebuild=False
):
"""Create a new library, and import documents"""
lib = Library(
self, import_fs, long_name=long_name, namespace=namespace, rebuild=rebuild
)
self.libs[lib.long_name] = lib
return lib
def finalize(self, ignore_errors=False):
self.build_libs(ignore_errors=ignore_errors)
for lib in itervalues(self.libs):
lib.on_archive_finalize()
if self.database_engines:
for app in itervalues(self.apps):
for model in app.lib.get_elements_by_type((namespaces.db, "model")):
if not model.abstract:
model.get_table_and_class(app)
def create_app(self, name, lib_name):
if name in self.apps:
raise errors.ArchiveError(
"Application name '{}' was previously installed with {}".format(
name, self.apps[name].lib
)
)
app = Application(self, name, lib_name)
self.apps[name] = app
app.settings.update(self.app_settings[name])
app.system_settings.update(self.app_system_settings[name])
self.apps_by_lib[lib_name].append(name)
return app
def get_app(self, app_id):
try:
return self.apps[app_id]
except KeyError:
return None
def has_app(self, app_id):
return app_id in self.apps
def get_app_from_lib(self, lib, current=None):
if isinstance(lib, Library):
lib_name = lib.long_name
else:
lib_name = text_type(lib)
if "." not in lib_name:
return lib_name
apps = self.apps_by_lib[lib_name]
if len(apps) != 1:
if not apps:
raise errors.AppRequiredError(lib_name)
if current:
if current.name in apps:
return current
raise errors.AmbiguousAppError(lib_name, apps)
return self.apps[apps[0]]
def get_app_from_lib_default(self, lib, default=None):
if isinstance(lib, Library):
lib_name = lib.long_name
else:
lib_name = text_type(lib)
if "." not in lib_name:
return lib_name
apps = self.apps_by_lib[lib_name]
if len(apps) != 1:
return default
return self.apps[apps[0]]
def find_app(self, name):
"""
Find an app from either its name or its lib name.
If a lib name is supplied and there are more than one app, an AmbiguousAppError is raise
"""
if isinstance(name, Application):
return name
if not name:
raise errors.UnknownAppError(
"Value {} is not a valid app or lib name".format(
to_expression(None, name)
)
)
name = text_type(name)
try:
if "." in name:
apps = self.apps_by_lib[name]
if not apps:
raise KeyError("No app called '{}'".format(name))
if len(apps) != 1:
raise errors.AmbiguousAppError(name, apps)
return self.apps[apps[0]]
else:
return self.apps[name]
except KeyError:
raise errors.UnknownAppError(app=name)
def find_app_default(self, name, default=None):
try:
return self.find_app(name)
except errors.UnknownAppError:
return default
def get_app_settings(self, name):
"""
Get settings object for an application.
"""
app = self.find_app(name)
return app.settings
def detect_app(self, context, name):
"""
Find an app from either its name or its libname
if the app is ambiguous, attempt to detect it from the callstack.
"""
if isinstance(name, Application):
return name
if not name:
raise errors.UnknownAppError(
"Value {} is not a valid app or lib name".format(context.expr(name))
)
try:
if "." in name:
apps = self.apps_by_lib[name]
if not apps:
raise KeyError("No app called '{}'".format(name))
if len(apps) != 1:
_app = context[".app"]
if _app and _app.name in apps:
return _app
for c in reversed(context["._callstack"]):
if c.app.name in apps:
return c.app
raise errors.AmbiguousAppError(name, apps)
return self.apps[apps[0]]
else:
return self.apps[name]
except KeyError:
raise errors.UnknownAppError(app=name)
def get_lib(self, name):
if isinstance(name, Library):
return name.long_name
if "." in name:
return self.libs[name].long_name
return self.find_app(name).lib.long_name
def add_data_tag(self, element_type, tag):
self.data_tags[element_type].append(tag)
self.data_tags_by_lib[tag.lib.long_name][element_type].append(tag)
def get_data(self, context, namespace, tag_name):
"""Get data from a data tag"""
tag_type = (namespace, tag_name)
return [
e.get_all_data_parameters(context)
for e in self.data_tags.get(tag_type, [])
if e.check(context)
]
def get_data_item(self, context, namespace, tag_name, filter_map, lib=None):
"""Get data from a data tag"""
tag_type = (namespace, tag_name)
for e in self.data_tags.get(tag_type, []):
if lib is not None:
if e.lib != lib:
continue
data = e.get_all_data_parameters(context)
if all(
filter_map[k] == data.get(k, Ellipsis) for k, v in filter_map.items()
):
return data
def get_data_from_element(self, context, element):
return element.get_all_data_parameters(context)
def get_data_elements(self, context, namespace, tag_name):
"""Get data from a data tag"""
tag_type = (namespace, tag_name)
return [
DataElement(e, context)
for e in self.data_tags.get(tag_type, [])
if e.check(context)
]
def get_app_data_elements(self, context, namespace, tag_name):
data = []
tag_type = (namespace, tag_name)
for app in itervalues(self.apps):
_elements = []
append = _elements.append
for e in self.data_tags_by_lib[app.lib.long_name].get(tag_type, []):
if e.check(context):
append(DataElement(e, context))
if _elements:
data.append((app, _elements))
return data
def add_filesystem(self, name, fs, create=False):
try:
add_fs = self.open_fs(fs, create=create)
except FSError as e:
raise errors.StartupFailedError(
"unable to open filesystem '{name}' ({e})".format(
name=name, e=text_type(e)
)
)
self.filesystems[name] = add_fs
# startup_log.debug("%s fs added as '%s'", add_fs, name)
startup_log.debug(
"<%s '%s'> filesystem added", type(add_fs).__name__.lower(), name
)
return fs
def get_filesystem(self, name):
return self.filesystems[name]
def has_filesystem(self, name):
return name in self.filesystems
def lookup_filesystem(self, element, name):
try:
return self.filesystems[name]
except KeyError:
raise element.throw(
"fs.no-filesystem",
"no filesystem called '{0}'".format(name),
diagnosis="You can view installed filesystems from the command line with **moya fs**",
)
def get_reader(self, name="data"):
fs = self.get_filesystem(name)
return DataReader(fs)
def get_context_filesystems(self):
return FSContainer((k, FSWrapper(fs)) for k, fs in iteritems(self.filesystems))
def get_translations(self, app_or_lib, languages):
return self.find_app(app_or_lib).lib.translations.get(languages)
def init_template_engine(self, system, settings):
if system in self.template_engines:
return
engine = TemplateEngine.create(system, self, self.templates_fs, settings)
self.template_engines[system] = engine
startup_log.debug("%s template engine initialized", engine)
def init_cache(self, name, settings):
cache = Cache.create(name, settings)
self.caches[name] = cache
startup_log.debug("%s cache added", cache)
def has_cache(self, name):
"""Check if a cache is present and enabled"""
if name not in self.caches:
return False
cache = self.caches[name]
return cache.enabled
def get_cache(self, name):
if name in self.caches:
return self.caches[name]
cache = self.caches[name] = Cache.create(
"runtime", SettingsSectionContainer({"type": "dict"})
)
return cache
def get_mailserver(self, name=None):
name = name or self.default_mail_server or "default"
try:
return self.mail_servers[name or "default"]
except KeyError:
raise errors.MoyaException(
"email.no-server", "no email server called '{0}'".format(name)
)
def init_templates(self, name, location, priority):
templates_fs = self.filesystems.get("templates")
fs = self.open_fs(location)
templates_fs.add_fs(name, fs, priority=priority)
# startup_log.debug("added templates filesystem, priority %s", priority)
def get_template_engine(self, engine="moya"):
return self.template_engines[engine]
def get_default_template_engine(self, app):
engine = app.lib.templates_info.get("default_engine", "moya")
return engine
def init_settings(self, cfg=None):
cfg = cfg or self.cfg
self.secret = cfg.get("project", "secret", "")
self.preflight = cfg.get_bool("project", "preflight", False)
self.debug = cfg.get_bool("project", "debug")
self.strict = self.strict or cfg.get_bool("project", "strict")
self.develop = self.develop or cfg.get_bool("project", "develop")
self.log_signals = cfg.get_bool("project", "log_signals")
self.debug_echo = cfg.get_bool("project", "debug_echo")
self.debug_memory = cfg.get_bool("project", "debug_memory")
self.lib_paths = cfg.get_list("project", "paths", "./local\n./external")
self.lib_paths = self.lib_paths[:] + [MOYA_LIBS_PATH]
if "console" in cfg:
self.log_logger = cfg.get("console", "logger", None)
self.log_color = cfg.get_bool("console", "color", True)
self.log_width = cfg.get_int("console", "width", None)
self.console = self.create_console()
self.sites.set_defaults(cfg["site"])
if "templates" not in self.caches:
self.caches["templates"] = Cache.create(
"templates", SettingsSectionContainer({"type": "dict"})
)
if "runtime" not in self.caches:
self.caches["runtime"] = Cache.create(
"runtime", SettingsSectionContainer({"type": "dict"})
)
require_name = ["app", "smtp", "db"]
self.auto_reload = cfg.get_bool("autoreload", "enabled")
if self.strict:
startup_log.debug("strict mode is enabled")
if self.develop:
startup_log.debug("develop mode is enabled")
for section_name, section in iteritems(cfg):
section = SectionWrapper(section_name, section)
if ":" in section_name:
what, name = section_name.split(":", 1)
else:
what = section_name
name = None
if what in require_name and not name:
raise errors.StartupFailedError(
"name required in section, [{section}:?]".format(section=what)
)
if what in ("project", "debug", "autoreload", "console", "customize", ""):
continue
if what == "settings":
if name is None:
self.settings.update(
(k, SettingContainer(v)) for k, v in iteritems(section)
)
else:
self.app_settings[name].update(
(k, SettingContainer(v)) for k, v in iteritems(section)
)
elif what == "application":
self.app_system_settings[name].update(section)
elif what == "lib":
if self.has_library(name):
lib = self.get_library(name)
lib.settings.update(
(k, SettingContainer(v)) for k, v in iteritems(section)
)
elif what == "fs":
location = section.get("location")
if not location:
raise errors.StartupFailedError(
"a value for 'location' is required in [{}]".format(
section_name
)
)
create = section.get_bool("create", False)
self.add_filesystem(name, location, create=create)
elif what == "data":
location = section.get("location")
data_fs = self.open_fs(location)
self.data_fs.add_fs(
"archive", data_fs, priority=section.get_int("priority", 0)
)
elif what == "cache":
self.init_cache(name, section)
elif what == "templates":
location = section["location"]
try:
priority = int(section["priority"])
except (IndexError, ValueError):
priority = 0
self.init_templates(name, location, priority)
elif what == "db":
from .db import add_engine
add_engine(self, name, section)
elif what == "media":
priority = section.get_int("priority", 1)
location = section["location"]
static_media_fs = self.open_fs(location)
media_fs = MultiFS()
media_fs.add_fs("static", static_media_fs, priority=priority)
self.add_filesystem("media", media_fs)
self.media_urls = section.get_list("url")
self.media_app = section.get("app", "media")
elif what == "smtp":
host = section["host"]
port = section.get_int("port", 25)
timeout = section.get_int("timeout", None)
username = section.get("username", None)
password = section.get("password", None)
default = section.get_bool("default", False)
sender = section.get("sender", None)
server = MailServer(
host,
name=name,
port=port,
default=default,
timeout=timeout,
username=username,
password=password,
sender=sender,
)
self.mail_servers[name] = server
if self.default_mail_server is None or default:
self.default_mail_server = name
if default:
startup_log.debug("%r (default) created", server)
else:
startup_log.debug("%r created", server)
elif what == "site":
if name:
self.sites.add_from_section(name, section)
elif what == "themes":
location = section["location"]
theme_fs = self.open_fs(location)
self.add_filesystem("themes", theme_fs)
# startup_log.debug("added theme filesystem '%s'", location)
else:
startup_log.warn("unknown settings section, [%s]", section_name)
self.init_template_engine("moya", {})
def init_media(self):
if "media" not in self.filesystems:
return
if not self.media_urls:
if not self.media_app:
raise errors.StartupFailedError(
"no 'url' or 'app' specified in [media] section"
)
if self.media_app not in self.apps:
startup_log.warning(
"app set in [media]/app has not been installed ({})".format(
self.media_app
)
)
return
try:
self.media_urls = [self.apps[self.media_app].mounts[0][1]]
except:
raise errors.StartupFailedError(
"unable to detect media url! (specify in [media]/url)"
)
for i, _url in enumerate(self.media_urls):
startup_log.debug("media url #%s is %s", i, _url)
media_fs = self.filesystems["media"]
media_mount_fs = MountFS()
for app in itervalues(self.apps):
for media_name, media_sub_fs in iteritems(app.lib.media):
name = "%s_%s" % (app.name, media_name)
media_path = "%s-%s" % (app.name, media_name)
app.media[media_name] = media_path
if name in self.filesystems:
mount_media = self.filesystems[name]
else:
mount_media = media_sub_fs
if name not in self.filesystems:
self.filesystems[name] = mount_media
media_mount_fs.mount(media_path, mount_media)
media_fs.add_fs("media", media_mount_fs)
def init_data(self):
data_fs = self.data_fs
for lib in itervalues(self.libs):
data_priority = lib.data_info.get("priority", 0)
if lib.data_priority is not None:
data_priority = lib.data_priority
if lib.data_fs is not None:
data_fs.add_fs(lib.long_name, lib.data_fs, priority=data_priority)
def get_media_url(self, context, app, media, path="", url_index=None):
"""Get a URL to media in a given app"""
if url_index is None:
url_index = context.inc("._media_url_index")
if not self.media_urls:
return None
url_no = url_index % len(self.media_urls)
if app is None:
return url_join(self.media_urls[url_no] or "", path)
return url_join(
self.media_urls[url_no] or "", app.get_media_directory(media), path
)
@property
def is_media_enabled(self):
"""Check if the media system is enabled"""
return bool(self.media_urls)
def get_element(self, element_ref, app=None, lib=None):
"""Gets an element from a reference"""
app_id, lib_id, name = self.parse_element_ref(element_ref)
if lib_id:
lib = self.get_library(lib_id)
element_app = self.get_app_from_lib_default(lib)
element = lib.get_named_element(name)
elif app_id:
try:
element_app = self.apps[app_id]
except KeyError:
raise errors.ElementNotFoundError(element_ref, app=app, lib=lib)
element = element_app.lib.get_named_element(name)
else:
if app is not None:
element_app = app
element = app.lib.get_named_element(name)
elif lib is not None:
element_app = app
element = lib.get_named_element(name)
else:
raise errors.ElementNotFoundError(element_ref, app=app, lib=lib)
if element is None:
raise errors.ElementNotFoundError(element_ref, app=app, lib=lib)
return FoundElement(element_app, element)
def get_app_element(self, element_ref):
element_app = self.find_app(element_ref.split("#", 1)[0])
app, element = self.get_element(element_ref, app=element_app)
return app or element_app, element
def resolve_template_path(self, path, app_name, base_path="/"):
"""Get a template path in the appropriate directory for an app"""
if path.startswith("/"):
return path
if isinstance(app_name, Application):
app = app_name
else:
app = self.find_app(text_type(app_name))
app = app or self.detect_app()
template_path = abspath(join(base_path, app.templates_directory, path))
return template_path
def get_template_lib(self, path, _cache=LRUCache()):
if path in _cache:
return _cache[path]
lib = None
for app in itervalues(self.apps):
if path.startswith(app.templates_directory):
lib = _cache[path] = app.lib.long_name
break
return lib
def get_elements_by_type(self, ns, type):
"""Get all elements of a given namespace, type"""
_elements = []
extend = _elements.extend
element_type = (ns, type)
for lib in itervalues(self.libs):
extend(lib.get_elements_by_type(element_type))
return _elements
def add_enum(self, libid, enum):
"""Add an enumeration"""
self.enum[libid] = enum
libname = libid.partition("#")[0]
if enum.name:
self.enum_by_lib.setdefault(libname, {})[enum.name] = enum
def get_enum(self, enum_libid):
"""Get an enumeration"""
return self.enum[enum_libid]
def get_lib_enums(self, libname):
"""Get enumeration for a given lib"""
return self.enum_by_lib.get(libname, None) or {}
def fire(self, context, signal_name, app=None, sender=None, data=None):
"""Fire a signal"""
if data is None:
data = {}
signal_obj = {"name": signal_name, "app": app, "sender": sender, "data": data}
if self.log_signals:
_params = context.to_expr(data)
if sender:
signal_log.debug(
'firing "%s" from "%s" %s', signal_name, sender, _params
)
else:
signal_log.debug('firing "%s" %s', signal_name, _params)
for element_ref in self.signals.filter_handlers(signal_name, sender):
_, libname, _ = self.parse_element_ref(element_ref)
for app_name in self.apps_by_lib[libname]:
app = self.apps[app_name]
try:
_callable = self.get_callable(element_ref, app=app)
_callable(context, signal=signal_obj)
except errors.LogicError as e:
# We can't risk any unhandled exceptions here
try:
log.error("%s unhandled in signal '%s'", e, signal_name)
except:
pass
try:
if context[".debug"]:
context[".console"].obj(context, e)
except:
pass
def get_callable(self, element_ref, app=None, breakpoint=False):
ref_app, element = self.get_element(element_ref, app=app)
if element is None:
raise errors.ElementNotFoundError(element_ref)
return CallableElement(self, element, ref_app or app, breakpoint=breakpoint)
def get_callable_from_element(self, element, app=None, breakpoint=False):
return CallableElement(self, element, app, breakpoint=breakpoint)
def call(self, element_ref, context, app, *args, **kwargs):
_callable = self.get_callable(element_ref, app=app)
return _callable(context, *args, **kwargs)
def call_params(self, element_ref, context, app, params):
_callable = self.get_callable(element_ref, app=app)
return _callable(context, **params)
def debug_call(self, element_ref, context, app, *args, **kwargs):
_callable = self.get_callable(element_ref, app=app, breakpoint=True)
return _callable(context, *args, **kwargs)
__call__ = call
class Archive(object):
_re_element_ref_match = re.compile(r'^(.+\..+)#(.*)$|^(.+)#(.*)$|^#(.+)$', re.UNICODE).match
def __init__(self, project_fs=None, breakpoint=False):
self.project_fs = project_fs
self.registry = ElementRegistry()
self.libs = {}
self.apps = OrderedDict()
self.apps_by_lib = defaultdict(list)
self.app_settings = defaultdict(SettingsContainer)
self.app_system_settings = defaultdict(SettingsContainer)
self.cfg = None
self.settings = SettingsContainer()
self.templates_fs = MultiFS()
self.data_fs = MultiFS()
self.filesystems = FSContainer({'templates': self.templates_fs,
'data': self.data_fs})
self.filters = FilterContainer(self)
self.template_engines = {}
self.database_engines = {}
self.caches = {}
self.mail_servers = {}
self.default_mail_server = None
self.default_db_engine = None
self.debug = False
self.auto_reload = False
self.known_namespaces = set()
self.sites = Sites()
self.breakpoint = breakpoint
self.suppress_breakpoints = False
self.data_tags = defaultdict(list)
self.data_tags_by_lib = defaultdict(lambda: defaultdict(list)) # awesome
self.preflight = False
self.log_signals = False
self.debug_echo = False
self.log_logger = None
self.log_color = True
self.log_width = None
self.media_urls = None
self.media_app = None
self.failed_documents = []
self.enum = {}
self.signals = Signals()
self._moyarc = None
self.console = self.create_console()
def __repr__(self):
return "<archive>"
@property
def media_url(self):
return self.media_urls[0] if self.media_urls else None
@property
def moyarc(self):
if self._moyarc is None:
try:
with io.open(os.path.expanduser("~/.moyarc"), 'rt') as f:
self._moyarc = settings.SettingsContainer.read_from_file(f)
except IOError:
self._moyarc = settings.SettingsContainer()
return self._moyarc
def open_fs(self, fs_url):
if isinstance(fs_url, text_type):
if '://' in fs_url:
fs = fsopendir(fs_url)
else:
fs = self.project_fs.opendir(fs_url)
else:
fs = fs_url
return fs
def get_console_file(self):
if self.log_logger:
console_file = logtools.LoggerFile(self.log_logger)
else:
console_file = None
return console_file
def create_console(self):
console = Console(out=self.get_console_file(),
nocolors=not (self.log_color and self.moyarc.get_bool('console', 'color', True)),
width=self.log_width or None)
return console
def build_libs(self, ignore_errors=False):
libs = [lib for lib in itervalues(self.libs) if not lib.built]
if not libs:
return
start = time()
self.build([doc for lib in libs
for doc in lib.documents],
log_time=False)
for lib in libs:
lib.finalize(ignore_errors=ignore_errors)
startup_log.debug("%s built libraries %0.1fms",
self,
(time() - start) * 1000.0)
def build(self, documents, context=None, log_time=True, fs=None):
"""Build all documents in the library"""
# This handles tags defined out of order
if fs is not None:
self.project_fs = fs
start = time()
if isinstance(documents, Document):
documents = [documents]
else:
documents = list(documents)
build_queue = deque()
build_queue_appendleft = build_queue.appendleft
for doc in documents:
if not doc.document_finalized and doc.structure:
build_queue.append(_BuildFrame([doc.structure.root_node]))
unbuildable = set()
unbuildable_clear = unbuildable.clear
unbuilt = []
if context is None:
context = Context()
context_root = context.root
get_doc_id = attrgetter('doc_id')
while build_queue:
nodes = build_queue[0]
if nodes:
node = nodes.pop()
context_root['_lib_long_name'] = node.lib_long_name
element = node.build(self, context)
if element:
unbuildable_clear()
build_queue_appendleft(_BuildFrame(sorted(node.children, key=get_doc_id, reverse=True), element=element))
else:
if element is not None:
if node in unbuildable:
# A previously deferred node, but we still can't built it
build_queue.popleft() # Can't process siblings either
unbuilt.append(node)
else:
# Defer this node till later
nodes.append(node)
build_queue.rotate(-1)
unbuildable.add(node)
else:
if nodes.element is not None:
unbuildable_clear()
try:
nodes.element.finalize(context)
except Exception as e:
raise
failed_doc = FailedDocument(path=node.structure.document.location,
code=node.structure.xml,
line=node.source_line,
col=None,
msg=text_type(e))
self.failed_documents.append(failed_doc)
#return False
build_queue.popleft()
if unbuilt:
for node in unbuilt:
nearest = nearest_word(node.tag_name,
self.registry.get_elements_in_xmlns(node.xmlns))
msg = "unknown tag {} in {}".format(node.tag_display_name, node.structure)
diagnosis = None
if nearest:
diagnosis = "did you mean <{}>?".format(nearest)
else:
find_xmlns = self.registry.find_xmlns(node.tag_name)
if find_xmlns:
diagnosis = "did you mean <{}> in XMLNS '{}'?".format(node.tag_name, find_xmlns)
failed_doc = FailedDocument(path=node.structure.document.location,
code=node.structure.xml,
line=node.source_line,
col=None,
msg=msg,
diagnosis=diagnosis)
self.failed_documents.append(failed_doc)
return False
for doc in documents:
doc.document_finalize(context)
if log_time:
doc_text = ', '.join(text_type(doc) for doc in documents)
startup_log.debug("%s built %0.1fms",
doc_text,
(time() - start) * 1000.0)
return True
def populate_context(self, context):
from .context.expressiontime import ExpressionDateTime
root = context.root
root['libs'] = self.libs
root['apps'] = self.apps
root['filters'] = self.filters
root['debug'] = self.debug
root['develop'] = self.develop
root['fs'] = self.get_context_filesystems()
root['now'] = ExpressionDateTime.utcnow()
from . import __version__
root['moya'] = {'version': __version__}
root['enum'] = self.enum
root['media_url'] = self.media_url
root['secret'] = self.secret
context.set_dynamic('.app', lambda context: getattr(context.get('.call', None), 'app'))
@classmethod
def get_callable_from_document(cls, path, element_ref, breakpoint=False, fs='./', default_context=False, archive=None, lib=None):
"""Shortcut that imports a single document and returns a callable"""
if archive is None:
archive = cls()
if lib is None:
lib = archive.create_library(long_name="moya.run", namespace=namespaces.run)
lib.import_document(fs, path)
archive.build_libs()
app, element = lib.documents[0].get_element(element_ref)
if element is None:
raise errors.ElementNotFoundError(element_ref, app=app)
call = CallableElement(archive, element, app, breakpoint=breakpoint)
if default_context:
def do_call(*args, **kwargs):
c = Context()
c['console'] = Console()
return call(c, *args, **kwargs)
do_call.archive = archive
return do_call
return call
@classmethod
def parse_element_ref(cls, s, cache={}):
try:
return cache[s]
except KeyError:
match = cls._re_element_ref_match(s)
if match is None:
result = None, None, s
else:
libname, lib_elementname, appname, app_elementname, docname = match.groups()
result = appname, libname, app_elementname or lib_elementname or docname
cache[s] = result
return result
def get_library(self, library_name):
"""Get a library from either its short name, or its long name"""
try:
return self.libs[library_name]
except KeyError:
raise errors.UnknownLibraryError(lib=library_name)
def has_library(self, library_name):
return library_name in self.libs
def load_library(self, import_fs, priority=None, template_priority=None, long_name=None, rebuild=False):
"""Load a new library in to the archive"""
lib = self.create_library(import_fs, long_name=long_name, rebuild=rebuild)
if priority is not None:
lib.priority = priority
if lib.templates_info:
fs_url = lib.templates_info['location']
if template_priority is None:
try:
template_priority = int(lib.templates_info.get('priority', '0'))
except ValueError:
startup_log.error("{} Invalid value for [templates]/priority, assuming 0".format(lib))
template_priority = 0
lib.template_priority = template_priority
if '://' in fs_url:
fs = fsopendir(fs_url)
else:
fs = import_fs.opendir(fs_url)
self.templates_fs.addfs(lib.long_name, fs, priority=template_priority)
return lib
def get_or_create_library(self, long_name, import_fs=None):
"""Get a library, or create it if it doesn't exists"""
if long_name is not None and long_name in self.libs:
return self.libs[long_name]
return self.create_library(import_fs)
def create_library(self, import_fs=None, long_name=None, namespace=None, rebuild=False):
"""Create a new library, and import documents"""
lib = Library(self, import_fs, long_name=long_name, namespace=namespace, rebuild=rebuild)
self.libs[lib.long_name] = lib
return lib
def finalize(self, ignore_errors=False):
self.build_libs(ignore_errors=ignore_errors)
for lib in itervalues(self.libs):
lib.on_archive_finalize()
if self.database_engines:
for app in itervalues(self.apps):
for model in app.lib.get_elements_by_type((namespaces.db, "model")):
if not model.abstract:
model.get_table_and_class(app)
def create_app(self, name, lib_name):
if name in self.apps:
raise errors.ArchiveError("Application name '{}' was previously installed with {}".format(name, self.apps[name].lib))
app = Application(self, name, lib_name)
self.apps[name] = app
app.settings.update(self.app_settings[name])
app.system_settings.update(self.app_system_settings[name])
self.apps_by_lib[lib_name].append(name)
return app
def get_app(self, app_id):
try:
return self.apps[app_id]
except IndexError:
return None
def has_app(self, app_id):
return app_id in self.apps
def get_app_from_lib(self, lib):
if isinstance(lib, Library):
lib_name = lib.long_name
else:
lib_name = text_type(lib)
if '.' not in lib_name:
return lib_name
apps = self.apps_by_lib[lib_name]
if len(apps) != 1:
if not apps:
raise errors.AppRequiredError(lib_name)
raise errors.AmbiguousAppError(lib_name, apps)
return self.apps[apps[0]]
def get_app_from_lib_default(self, lib, default=None):
if isinstance(lib, Library):
lib_name = lib.long_name
else:
lib_name = text_type(lib)
if '.' not in lib_name:
return lib_name
apps = self.apps_by_lib[lib_name]
if len(apps) != 1:
return default
return self.apps[apps[0]]
def find_app(self, name):
"""Find an app from either its name or its lib name.
If a lib name is supplied and there are more than one app, an AmbiguousAppError is raise
"""
if isinstance(name, Application):
return name
if not name:
raise errors.UnknownAppError("Value {} is not a valid app or lib name".format(to_expression(None, name)))
name = text_type(name)
try:
if '.' in name:
apps = self.apps_by_lib[name]
if not apps:
raise KeyError("No app called '{}'".format(name))
if len(apps) != 1:
raise errors.AmbiguousAppError(name, apps)
return self.apps[apps[0]]
else:
return self.apps[name]
except KeyError:
raise errors.UnknownAppError(app=name)
def get_lib(self, name):
if isinstance(name, Library):
return name.long_name
if '.' in name:
return self.libs[name].long_name
return self.find_app(name).lib.long_name
def add_data_tag(self, element_type, tag):
self.data_tags[element_type].append(tag)
self.data_tags_by_lib[tag.lib.long_name][element_type].append(tag)
def get_data(self, context, namespace, tag_name):
"""Get data from a data tag"""
tag_type = (namespace, tag_name)
return [e.get_all_data_parameters(context)
for e in self.data_tags.get(tag_type, [])
if e.check(context)]
def get_data_item(self, context, namespace, tag_name, filter_map):
"""Get data from a data tag"""
tag_type = (namespace, tag_name)
for e in self.data_tags.get(tag_type, []):
data = e.get_all_data_parameters(context)
if all(filter_map[k] == data.get(k, Ellipsis)
for k, v in filter_map.items()):
return data
def get_data_from_element(self, context, element):
return element.get_all_data_parameters(context)
def get_data_elements(self, context, namespace, tag_name):
"""Get data from a data tag"""
tag_type = (namespace, tag_name)
return [DataElement(e, context)
for e in self.data_tags.get(tag_type, [])
if e.check(context)]
def get_app_data_elements(self, context, namespace, tag_name):
data = []
tag_type = (namespace, tag_name)
for app in itervalues(self.apps):
_elements = []
append = _elements.append
for e in self.data_tags_by_lib[app.lib.long_name].get(tag_type, []):
if e.check(context):
append(DataElement(e, context))
if _elements:
data.append((app, _elements))
return data
def add_filesystem(self, name, fs):
add_fs = self.open_fs(fs)
self.filesystems[name] = add_fs
startup_log.debug("%s fs added as '%s'", add_fs, name)
return fs
def get_filesystem(self, name):
return self.filesystems[name]
def get_reader(self, name="data"):
fs = self.get_filesystem(name)
return DataReader(fs)
def get_context_filesystems(self):
return FSContainer((k, FSWrapper(fs))
for k, fs in iteritems(self.filesystems))
def get_translations(self, app_or_lib, languages):
return self.find_app(app_or_lib).lib.translations.get(languages)
def init_template_engine(self, system, settings):
if system in self.template_engines:
return
engine = TemplateEngine.create(system,
self,
self.templates_fs,
settings)
self.template_engines[system] = engine
startup_log.debug('%s template engine initialized', engine)
def init_cache(self, name, settings):
cache = Cache.create(name, settings)
self.caches[name] = cache
startup_log.debug('%s cache added', cache)
def get_cache(self, name):
if name in self.caches:
return self.caches[name]
cache = self.caches[name] = Cache.create('runtime', SettingsSectionContainer({'type': 'dict'}))
return cache
def get_mailserver(self, name=None):
name = name or self.default_mail_server or 'default'
try:
return self.mail_servers[name or 'default']
except KeyError:
raise errors.MoyaException("email.no-server", "no email server called '{0}'".format(name))
def init_templates(self, name, location, priority):
templates_fs = self.filesystems.get("templates")
fs = self.open_fs(location)
templates_fs.addfs(name, fs, priority=priority)
startup_log.debug("%s added to templates filesystem, priority %s", fs, priority)
def get_template_engine(self, engine="moya"):
return self.template_engines[engine]
def get_default_template_engine(self, app):
engine = app.lib.templates_info.get('default_engine', 'moya')
return engine
def init_settings(self, cfg=None):
cfg = cfg or self.cfg
self.secret = cfg.get('project', 'secret', '')
self.preflight = cfg.get_bool('project', 'preflight', False)
self.debug = cfg.get_bool('project', 'debug')
self.develop = cfg.get_bool('project', 'develop')
self.log_signals = cfg.get_bool('project', 'log_signals')
self.debug_echo = cfg.get_bool('project', 'debug_echo')
if 'console' in cfg:
self.log_logger = cfg.get('console', 'logger', None)
self.log_color = cfg.get_bool('console', 'color', True)
self.log_width = cfg.get_int('console', 'width', None)
self.console = self.create_console()
self.sites.set_defaults(cfg['site'])
if 'templates' not in self.caches:
self.caches['templates'] = Cache.create('templates', SettingsSectionContainer({'type': 'dict'}))
if 'runtime' not in self.caches:
self.caches['runtime'] = Cache.create('runtime', SettingsSectionContainer({'type': 'dict'}))
require_name = ['app', 'smtp', 'db']
self.auto_reload = cfg.get_bool('autoreload', 'enabled')
for section_name, section in iteritems(cfg):
section = SectionWrapper(section_name, section)
if ':' in section_name:
what, name = section_name.split(':', 1)
else:
what = section_name
name = None
if what in require_name and not name:
raise errors.StartupFailedError('Name/text required in project settings [{section}:]'.format(section=what))
if what in ('project', 'debug', 'autoreload', 'console', ''):
continue
if what == "settings":
if name is None:
self.settings.update((k, SettingContainer(v))
for k, v in iteritems(section))
else:
self.app_settings[name].update((k, SettingContainer(v))
for k, v in iteritems(section))
elif what == 'application':
self.app_system_settings[name].update(section)
elif what == "lib":
if self.has_library(name):
lib = self.get_library(name)
lib.settings.update((k, SettingContainer(v))
for k, v in iteritems(section))
elif what == "fs":
location = section.get("location")
self.add_filesystem(name, location)
elif what == "data":
location = section.get("location")
data_fs = self.open_fs(location)
self.data_fs.addfs('archive',
data_fs,
priority=section.get_int('priority', 0))
elif what == "cache":
self.init_cache(name, section)
elif what == "templates":
location = section["location"]
try:
priority = int(section["priority"])
except (IndexError, ValueError):
priority = 0
self.init_templates(name, location, priority)
elif what == "db":
from .db import add_engine
add_engine(self, name, section)
elif what == 'media':
priority = section.get_int('priority', 1)
location = section["location"]
static_media_fs = self.open_fs(location)
media_fs = MultiFS()
media_fs.addfs("static", static_media_fs, priority=priority)
self.add_filesystem('media', media_fs)
self.media_urls = section.get_list('url')
self.media_app = section.get('app', 'media')
elif what == "smtp":
host = section["host"]
port = section.get_int('port', 25)
timeout = section.get_int('timeout', None)
username = section.get('username', None)
password = section.get("password", None)
default = section.get_bool('default', False)
sender = section.get('sender', None)
server = MailServer(host,
name=name,
port=port,
default=default,
timeout=timeout,
username=username,
password=password,
sender=sender)
self.mail_servers[name] = server
if self.default_mail_server is None or default:
self.default_mail_server = name
if default:
startup_log.debug('%r (default) created', server)
else:
startup_log.debug('%r created', server)
elif what == "site":
if name:
self.sites.add_from_section(name, section)
else:
startup_log.warn("unknown settings section: [%s]", section_name)
self.init_template_engine('moya', {})
def init_media(self):
if 'media' not in self.filesystems:
return
if not self.media_urls:
if not self.media_app:
raise errors.StartupFailedError("no 'url' or 'app' specified in [media] section")
if self.media_app not in self.apps:
startup_log.warning('app set in [media]/app has not been installed ({})'.format(self.media_app))
return
try:
self.media_urls = [self.apps[self.media_app].mounts[0][1]]
except:
raise errors.StartupFailedError('unable to detect media url! (specify in [media]/url)')
for i, _url in enumerate(self.media_urls):
startup_log.debug('media url #%s is %s', i, _url)
media_fs = self.filesystems['media']
media_mount_fs = MountFS()
for app in itervalues(self.apps):
for media_name, media_sub_fs in iteritems(app.lib.media):
name = "%s_%s" % (app.name, media_name)
media_path = "%s-%s" % (app.name, media_name)
app.media[media_name] = media_path
if name in self.filesystems:
mount_media = self.filesystems[name]
else:
mount_media = media_sub_fs
if name not in self.filesystems:
self.filesystems[name] = mount_media
media_mount_fs.mountdir(media_path, mount_media)
media_fs.addfs("media", media_mount_fs)
def init_data(self):
data_fs = self.data_fs
for lib in itervalues(self.libs):
if lib.data_fs is not None:
data_fs.addfs(lib.long_name, lib.data_fs, priority=lib.data_info.get('priorty', 0))
def get_media_url(self, context, app, media, path='', url_index=None):
"""Get a URL to media in a given app"""
if url_index is None:
url_index = context.inc('._media_url_index')
url_no = url_index % len(self.media_urls)
if app is None:
return url_join(self.media_urls[url_no] or '', path)
return url_join(self.media_urls[url_no] or '',
app.get_media_directory(media),
path)
def get_element(self, element_ref, app=None, lib=None):
"""Gets an element from a reference"""
app_id, lib_id, name = self.parse_element_ref(element_ref)
if lib_id:
lib = self.get_library(lib_id)
element_app = self.get_app_from_lib_default(lib)
element = lib.get_named_element(name)
elif app_id:
try:
element_app = self.apps[app_id]
except KeyError:
raise errors.ElementNotFoundError(element_ref, app=app, lib=lib)
element = element_app.lib.get_named_element(name)
else:
if app is not None:
element_app = app
element = app.lib.get_named_element(name)
elif lib is not None:
element_app = app
element = lib.get_named_element(name)
else:
raise errors.ElementNotFoundError(element_ref, app=app, lib=lib)
if element is None:
raise errors.ElementNotFoundError(element_ref, app=app, lib=lib)
return FoundElement(element_app, element)
def get_app_element(self, element_ref):
element_app = self.find_app(element_ref.split('#', 1)[0])
app, element = self.get_element(element_ref, app=element_app)
return app or element_app, element
def resolve_template_path(self, path, app_name, base_path="/"):
"""Get a template path in the appropriate directory for an app"""
if path.startswith('/'):
return path
if isinstance(app_name, Application):
app = app_name
else:
app = self.find_app(text_type(app_name))
template_path = abspath(pathjoin(base_path, app.templates_directory, path))
return template_path
def get_template_lib(self, path, _cache=LRUCache()):
if path in _cache:
return _cache[path]
lib = None
for app in itervalues(self.apps):
if path.startswith(app.templates_directory):
lib = _cache[path] = app.lib.long_name
break
return lib
def get_elements_by_type(self, ns, type):
"""Get all elements of a given namespace, type"""
_elements = []
extend = _elements.extend
element_type = (ns, type)
for lib in itervalues(self.libs):
extend(lib.get_elements_by_type(element_type))
return _elements
def add_enum(self, enum):
"""Add an enumeration"""
self.enum[enum.name] = enum
def get_enum(self, enum_libid):
"""Get an enumeration"""
return self.enum[enum_libid]
def fire(self, context, signal_name, app=None, sender=None, data=None):
"""Fire a signal"""
if data is None:
data = {}
signal_obj = {
"name": signal_name,
"app": app,
"sender": sender,
"data": data
}
if self.log_signals:
_params = context.to_expr(data)
if sender:
signal_log.debug('firing "%s" from "%s" %s', signal_name, sender, _params)
else:
signal_log.debug('firing "%s" %s', signal_name, _params)
for element_ref in self.signals.filter_handlers(signal_name, sender):
_, libname, _ = self.parse_element_ref(element_ref)
for app_name in self.apps_by_lib[libname]:
app = self.apps[app_name]
try:
_callable = self.get_callable(element_ref, app=app)
_callable(context, signal=signal_obj)
except errors.LogicError as e:
# We can't risk any unhandled exceptions here
try:
log.error("%s unhandled in signal '%s'", e, signal_name)
except:
pass
try:
if context['.debug']:
context['.console'].obj(context, e)
except:
pass
def get_callable(self, element_ref, app=None, breakpoint=False):
ref_app, element = self.get_element(element_ref, app=app)
if element is None:
raise errors.ElementNotFoundError(element_ref)
return CallableElement(self, element, ref_app or app, breakpoint=breakpoint)
def get_callable_from_element(self, element, app=None, breakpoint=False):
return CallableElement(self, element, app, breakpoint=breakpoint)
def call(self, element_ref, context, app, *args, **kwargs):
_callable = self.get_callable(element_ref, app=app)
return _callable(context, *args, **kwargs)
def debug_call(self, element_ref, context, app, *args, **kwargs):
_callable = self.get_callable(element_ref, app=app, breakpoint=True)
return _callable(context, *args, **kwargs)
__call__ = call
def _get_fs(self):
mfs = MultiFS()
profiles = get_blob_fs("profiles")
mfs.add_fs("profile_fs", profiles, priority=2)
mfs.add_fs("local_fs", self.local_fs, write=True, priority=3)
return mfs
def build(
fs,
settings_path="settings.ini",
rebuild=False,
archive=None,
strict=False,
master_settings=None,
test_build=False,
develop=False,
):
"""Build a project"""
if isinstance(fs, string_types):
if "://" in fs:
fs = open_fs(fs)
else:
fs = OSFS(fs)
if isinstance(settings_path, string_types):
settings_path = [settings_path]
try:
syspath = fs.getsyspath("/")
except NoSysPath:
syspath = None
cwd = os.getcwd()
if syspath is not None:
os.chdir(syspath)
try:
log.debug("reading settings from {}".format(
textual_list(settings_path)))
cfg = SettingsContainer.read(fs, settings_path, master=master_settings)
if "customize" in cfg:
customize_location = cfg.get("customize", "location")
if customize_location:
settings_path = cfg.get("customize", "settings",
"settings.ini")
startup_log.info("customizing '%s'", customize_location)
customize_fs = open_fs(cfg.get("customize", "location"))
cfg = SettingsContainer.read(customize_fs,
settings_path,
master=cfg)
overlay_fs = MultiFS()
overlay_fs.add_fs("project", fs)
overlay_fs.add_fs("custom", customize_fs, write=True)
fs = overlay_fs
try:
syspath = fs.getsyspath("/")
except NoSysPath:
pass
else:
os.chdir(syspath)
if archive is None:
archive = Archive(fs,
strict=strict,
test_build=test_build,
develop=develop)
context = Context()
archive.cfg = cfg
root = context.root
root["libs"] = archive.libs
root["apps"] = archive.apps
root["fs"] = FSWrapper(fs)
root["settings"] = SettingsContainer.from_dict(archive.cfg["settings"])
startup_path = archive.cfg.get("project", "startup")
docs_location = archive.cfg.get("project", "location")
archive.init_settings()
root["console"] = archive.console
root["debug"] = archive.debug
root["_rebuild"] = rebuild
parser = Parser(archive, fs.opendir(docs_location), startup_path)
doc = parser.parse()
if doc is None:
raise errors.StartupFailedError(
'unable to parse "{}"'.format(startup_path))
archive.build(doc, fs=fs)
return fs, archive, context, doc
finally:
os.chdir(cwd)
gc.collect()
def get_multi_fs(directories):
filesystem = MultiFS()
for directory in directories:
filesystem.add_fs(directory, fs.open_fs(directory))
return filesystem
def test_no_writable(self):
fs = MultiFS()
with self.assertRaises(errors.ResourceReadOnly):
fs.setbytes('foo', b'bar')
def test_priority(self):
"""Test priority order is working"""
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", b("m1"))
m2.setcontents("name", b("m2"))
m3.setcontents("name", b("m3"))
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2)
multi_fs.addfs("m3", m3)
self.assert_(multi_fs.getcontents("name") == b("m3"))
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", b("m1"))
m2.setcontents("name", b("m2"))
m3.setcontents("name", b("m3"))
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3)
self.assert_(multi_fs.getcontents("name") == b("m2"))
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", b("m1"))
m2.setcontents("name", b("m2"))
m3.setcontents("name", b("m3"))
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3, priority=10)
self.assert_(multi_fs.getcontents("name") == b("m3"))
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", b("m1"))
m2.setcontents("name", b("m2"))
m3.setcontents("name", b("m3"))
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1, priority=11)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3, priority=10)
self.assert_(multi_fs.getcontents("name") == b("m1"))
def init_settings(self, cfg=None):
cfg = cfg or self.cfg
self.secret = cfg.get('project', 'secret', '')
self.preflight = cfg.get_bool('project', 'preflight', False)
self.debug = cfg.get_bool('project', 'debug')
self.develop = cfg.get_bool('project', 'develop')
self.log_signals = cfg.get_bool('project', 'log_signals')
self.debug_echo = cfg.get_bool('project', 'debug_echo')
if 'console' in cfg:
self.log_logger = cfg.get('console', 'logger', None)
self.log_color = cfg.get_bool('console', 'color', True)
self.log_width = cfg.get_int('console', 'width', None)
self.console = self.create_console()
self.sites.set_defaults(cfg['site'])
if 'templates' not in self.caches:
self.caches['templates'] = Cache.create('templates', SettingsSectionContainer({'type': 'dict'}))
if 'runtime' not in self.caches:
self.caches['runtime'] = Cache.create('runtime', SettingsSectionContainer({'type': 'dict'}))
require_name = ['app', 'smtp', 'db']
self.auto_reload = cfg.get_bool('autoreload', 'enabled')
for section_name, section in iteritems(cfg):
section = SectionWrapper(section_name, section)
if ':' in section_name:
what, name = section_name.split(':', 1)
else:
what = section_name
name = None
if what in require_name and not name:
raise errors.StartupFailedError('Name/text required in project settings [{section}:]'.format(section=what))
if what in ('project', 'debug', 'autoreload', 'console', ''):
continue
if what == "settings":
if name is None:
self.settings.update((k, SettingContainer(v))
for k, v in iteritems(section))
else:
self.app_settings[name].update((k, SettingContainer(v))
for k, v in iteritems(section))
elif what == 'application':
self.app_system_settings[name].update(section)
elif what == "lib":
if self.has_library(name):
lib = self.get_library(name)
lib.settings.update((k, SettingContainer(v))
for k, v in iteritems(section))
elif what == "fs":
location = section.get("location")
self.add_filesystem(name, location)
elif what == "data":
location = section.get("location")
data_fs = self.open_fs(location)
self.data_fs.addfs('archive',
data_fs,
priority=section.get_int('priority', 0))
elif what == "cache":
self.init_cache(name, section)
elif what == "templates":
location = section["location"]
try:
priority = int(section["priority"])
except (IndexError, ValueError):
priority = 0
self.init_templates(name, location, priority)
elif what == "db":
from .db import add_engine
add_engine(self, name, section)
elif what == 'media':
priority = section.get_int('priority', 1)
location = section["location"]
static_media_fs = self.open_fs(location)
media_fs = MultiFS()
media_fs.addfs("static", static_media_fs, priority=priority)
self.add_filesystem('media', media_fs)
self.media_urls = section.get_list('url')
self.media_app = section.get('app', 'media')
elif what == "smtp":
host = section["host"]
port = section.get_int('port', 25)
timeout = section.get_int('timeout', None)
username = section.get('username', None)
password = section.get("password", None)
default = section.get_bool('default', False)
sender = section.get('sender', None)
server = MailServer(host,
name=name,
port=port,
default=default,
timeout=timeout,
username=username,
password=password,
sender=sender)
self.mail_servers[name] = server
if self.default_mail_server is None or default:
self.default_mail_server = name
if default:
startup_log.debug('%r (default) created', server)
else:
startup_log.debug('%r created', server)
elif what == "site":
if name:
self.sites.add_from_section(name, section)
else:
startup_log.warn("unknown settings section: [%s]", section_name)
self.init_template_engine('moya', {})
def test_priority(self):
"""Test priority order is working"""
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.writebytes("name", b"m1")
m2.writebytes("name", b"m2")
m3.writebytes("name", b"m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.add_fs("m1", m1)
multi_fs.add_fs("m2", m2)
multi_fs.add_fs("m3", m3)
self.assertEqual(multi_fs.readbytes("name"), b"m3")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.writebytes("name", b"m1")
m2.writebytes("name", b"m2")
m3.writebytes("name", b"m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.add_fs("m1", m1)
multi_fs.add_fs("m2", m2, priority=10)
multi_fs.add_fs("m3", m3)
self.assertEqual(multi_fs.readbytes("name"), b"m2")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.writebytes("name", b"m1")
m2.writebytes("name", b"m2")
m3.writebytes("name", b"m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.add_fs("m1", m1)
multi_fs.add_fs("m2", m2, priority=10)
multi_fs.add_fs("m3", m3, priority=10)
self.assertEqual(multi_fs.readbytes("name"), b"m3")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.writebytes("name", b"m1")
m2.writebytes("name", b"m2")
m3.writebytes("name", b"m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.add_fs("m1", m1, priority=11)
multi_fs.add_fs("m2", m2, priority=10)
multi_fs.add_fs("m3", m3, priority=10)
self.assertEqual(multi_fs.readbytes("name"), b"m1")
def open_file(
authority,
cache,
update,
version_check,
hasher,
read_path,
write_path=None,
cache_on_write=False,
mode='r',
*args,
**kwargs):
'''
Context manager for reading/writing an archive and uploading on changes
Parameters
----------
authority : object
:py:mod:`pyFilesystem` filesystem object to use as the authoritative,
up-to-date source for the archive
cache : object
:py:mod:`pyFilesystem` filesystem object to use as the cache. Default
``None``.
use_cache : bool
update, service_path, version_check, \*\*kwargs
'''
if write_path is None:
write_path = read_path
with _choose_read_fs(
authority, cache, read_path, version_check, hasher) as read_fs:
write_mode = ('w' in mode) or ('a' in mode) or ('+' in mode)
if write_mode:
readwrite_mode = (
('a' in mode) or (
('r' in mode) and (
'+' in mode)))
with _prepare_write_fs(
read_fs, cache, read_path, readwrite_mode) as write_fs:
wrapper = MultiFS()
wrapper.addfs('reader', read_fs)
wrapper.setwritefs(write_fs)
with wrapper.open(read_path, mode, *args, **kwargs) as f:
yield f
info = write_fs.getinfokeys(read_path, 'size')
if 'size' in info:
if info['size'] == 0:
return
with write_fs.open(read_path, 'rb') as f:
checksum = hasher(f)
if not version_check(checksum):
if (
cache_on_write or
(
cache
and (
fs.path.abspath(read_path) ==
fs.path.abspath(write_path))
and cache.fs.isfile(read_path)
)
):
_makedirs(cache.fs, fs.path.dirname(write_path))
fs.utils.copyfile(
write_fs, read_path, cache.fs, write_path)
_makedirs(authority.fs, fs.path.dirname(write_path))
fs.utils.copyfile(
cache.fs, write_path, authority.fs, write_path)
else:
_makedirs(authority.fs, fs.path.dirname(write_path))
fs.utils.copyfile(
write_fs, read_path, authority.fs, write_path)
update(**checksum)
else:
with read_fs.open(read_path, mode, *args, **kwargs) as f:
yield f
def setUp(self):
fs = MultiFS()
mem_fs = MemoryFS()
fs.add_fs("mem", mem_fs, write=True)
self.fs = fs
self.mem_fs = mem_fs
def test_priority(self):
"""Test priority order is working"""
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2)
multi_fs.addfs("m3", m3)
self.assert_(multi_fs.getcontents("name") == "m3")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3)
self.assert_(multi_fs.getcontents("name") == "m2")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3, priority=10)
self.assert_(multi_fs.getcontents("name") == "m3")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1, priority=11)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3, priority=10)
self.assert_(multi_fs.getcontents("name") == "m1")
def test_no_writable(self):
fs = MultiFS()
with self.assertRaises(errors.ResourceReadOnly):
fs.writebytes("foo", b"bar")
def __init__(self, **kwargs): self.fs = MultiFS() self.config = kwargs['config'] self.basepath = kwargs['basepath']
def init_settings(self, cfg=None):
cfg = cfg or self.cfg
self.secret = cfg.get("project", "secret", "")
self.preflight = cfg.get_bool("project", "preflight", False)
self.debug = cfg.get_bool("project", "debug")
self.strict = self.strict or cfg.get_bool("project", "strict")
self.develop = self.develop or cfg.get_bool("project", "develop")
self.log_signals = cfg.get_bool("project", "log_signals")
self.debug_echo = cfg.get_bool("project", "debug_echo")
self.debug_memory = cfg.get_bool("project", "debug_memory")
self.lib_paths = cfg.get_list("project", "paths", "./local\n./external")
self.lib_paths = self.lib_paths[:] + [MOYA_LIBS_PATH]
if "console" in cfg:
self.log_logger = cfg.get("console", "logger", None)
self.log_color = cfg.get_bool("console", "color", True)
self.log_width = cfg.get_int("console", "width", None)
self.console = self.create_console()
self.sites.set_defaults(cfg["site"])
if "templates" not in self.caches:
self.caches["templates"] = Cache.create(
"templates", SettingsSectionContainer({"type": "dict"})
)
if "runtime" not in self.caches:
self.caches["runtime"] = Cache.create(
"runtime", SettingsSectionContainer({"type": "dict"})
)
require_name = ["app", "smtp", "db"]
self.auto_reload = cfg.get_bool("autoreload", "enabled")
if self.strict:
startup_log.debug("strict mode is enabled")
if self.develop:
startup_log.debug("develop mode is enabled")
for section_name, section in iteritems(cfg):
section = SectionWrapper(section_name, section)
if ":" in section_name:
what, name = section_name.split(":", 1)
else:
what = section_name
name = None
if what in require_name and not name:
raise errors.StartupFailedError(
"name required in section, [{section}:?]".format(section=what)
)
if what in ("project", "debug", "autoreload", "console", "customize", ""):
continue
if what == "settings":
if name is None:
self.settings.update(
(k, SettingContainer(v)) for k, v in iteritems(section)
)
else:
self.app_settings[name].update(
(k, SettingContainer(v)) for k, v in iteritems(section)
)
elif what == "application":
self.app_system_settings[name].update(section)
elif what == "lib":
if self.has_library(name):
lib = self.get_library(name)
lib.settings.update(
(k, SettingContainer(v)) for k, v in iteritems(section)
)
elif what == "fs":
location = section.get("location")
if not location:
raise errors.StartupFailedError(
"a value for 'location' is required in [{}]".format(
section_name
)
)
create = section.get_bool("create", False)
self.add_filesystem(name, location, create=create)
elif what == "data":
location = section.get("location")
data_fs = self.open_fs(location)
self.data_fs.add_fs(
"archive", data_fs, priority=section.get_int("priority", 0)
)
elif what == "cache":
self.init_cache(name, section)
elif what == "templates":
location = section["location"]
try:
priority = int(section["priority"])
except (IndexError, ValueError):
priority = 0
self.init_templates(name, location, priority)
elif what == "db":
from .db import add_engine
add_engine(self, name, section)
elif what == "media":
priority = section.get_int("priority", 1)
location = section["location"]
static_media_fs = self.open_fs(location)
media_fs = MultiFS()
media_fs.add_fs("static", static_media_fs, priority=priority)
self.add_filesystem("media", media_fs)
self.media_urls = section.get_list("url")
self.media_app = section.get("app", "media")
elif what == "smtp":
host = section["host"]
port = section.get_int("port", 25)
timeout = section.get_int("timeout", None)
username = section.get("username", None)
password = section.get("password", None)
default = section.get_bool("default", False)
sender = section.get("sender", None)
server = MailServer(
host,
name=name,
port=port,
default=default,
timeout=timeout,
username=username,
password=password,
sender=sender,
)
self.mail_servers[name] = server
if self.default_mail_server is None or default:
self.default_mail_server = name
if default:
startup_log.debug("%r (default) created", server)
else:
startup_log.debug("%r created", server)
elif what == "site":
if name:
self.sites.add_from_section(name, section)
elif what == "themes":
location = section["location"]
theme_fs = self.open_fs(location)
self.add_filesystem("themes", theme_fs)
# startup_log.debug("added theme filesystem '%s'", location)
else:
startup_log.warn("unknown settings section, [%s]", section_name)
self.init_template_engine("moya", {})
def _get_fs(self, fs_url):
mfs = MultiFS()
mfs.add_fs("remotefs", fs.open_fs(fs_url), write=True, priority=3)
return mfs
def make_fs(self):
fs = MultiFS()
mem_fs = MemoryFS()
fs.add_fs("mem", mem_fs, write=True)
return fs
