def test_listdir_duplicates(self): m1 = MemoryFS() m2 = MemoryFS() m1.touch("foo") m2.touch("foo") multi_fs = MultiFS() multi_fs.add_fs("m1", m1) multi_fs.add_fs("m2", m2) self.assertEqual(multi_fs.listdir("/"), ["foo"])
def test_listdir_duplicates(self): m1 = MemoryFS() m2 = MemoryFS() m1.touch('foo') m2.touch('foo') multi_fs = MultiFS() multi_fs.add_fs('m1', m1) multi_fs.add_fs('m2', m2) self.assertEqual(multi_fs.listdir(u'/'), ['foo'])
def __init__(self, project_fs=None, breakpoint=False, strict=False, test_build=False, develop=False): self.project_fs = project_fs self.strict = strict self.test_build = test_build self.develop = develop self.registry = ElementRegistry() self.libs = {} self.apps = OrderedDict() self.apps_by_lib = defaultdict(list) self.app_settings = defaultdict(SettingsContainer) self.app_system_settings = defaultdict(SettingsContainer) self.cfg = None self.settings = SettingsContainer() self.templates_fs = MultiFS() self.data_fs = MultiFS() self.filesystems = FSContainer({'templates': self.templates_fs, 'data': self.data_fs}) self.filters = FilterContainer(self) self.template_engines = {} self.database_engines = {} self.caches = {} self.mail_servers = {} self.default_mail_server = None self.default_db_engine = None self.debug = False self.struct = False self.auto_reload = False self.known_namespaces = set() self.sites = Sites() self.breakpoint = breakpoint self.suppress_breakpoints = False self.data_tags = defaultdict(list) self.data_tags_by_lib = defaultdict(lambda: defaultdict(list)) # awesome self.preflight = False self.log_signals = False self.debug_echo = False self.debug_memory = False self.lib_paths = None self._lib_database = None self.log_logger = None self.log_color = True self.log_width = None self.media_urls = None self.media_app = None self.failed_documents = [] self.enum = {} self.enum_by_lib = {} self.signals = Signals() self._moyarc = None self.console = self.create_console()
def test_opener(self): """Test use of FS URLs.""" multi_fs = MultiFS() with self.assertRaises(TypeError): multi_fs.add_fs(u'foo', 5) multi_fs.add_fs(u'f1', u'mem://') multi_fs.add_fs(u'f2', u'temp://') self.assertIsInstance(multi_fs.get_fs(u'f1'), MemoryFS)
def test_opener(self): """Test use of FS URLs.""" multi_fs = MultiFS() with self.assertRaises(TypeError): multi_fs.add_fs("foo", 5) multi_fs.add_fs("f1", "mem://") multi_fs.add_fs("f2", "temp://") self.assertIsInstance(multi_fs.get_fs("f1"), MemoryFS)
def test_auto_close(self): """Test MultiFS auto close is working""" multi_fs = MultiFS() m1 = MemoryFS() m2 = MemoryFS() multi_fs.addfs('m1', m1) multi_fs.addfs('m2', m2) self.assert_(not m1.closed) self.assert_(not m2.closed) multi_fs.close() self.assert_(m1.closed) self.assert_(m2.closed)
def test_no_auto_close(self): """Test MultiFS auto close can be disables""" multi_fs = MultiFS(auto_close=False) m1 = MemoryFS() m2 = MemoryFS() multi_fs.addfs('m1', m1) multi_fs.addfs('m2', m2) self.assert_(not m1.closed) self.assert_(not m2.closed) multi_fs.close() self.assert_(not m1.closed) self.assert_(not m2.closed)
def _find_extensions(paths): '''Iterate the paths, in order, finding extensions and adding them to the return dict.''' extension_kinds = ['check', 'configure', 'write'] efs = MultiFS() map(lambda x: efs.addfs(x, OSFS(x)), paths) def get_extensions(kind): return {os.path.splitext(x)[0]: efs.getsyspath(x) for x in efs.walkfiles('.', '*.%s' % kind)} return {e: get_extensions(e) for e in extension_kinds}
class _AnsiblePlaybookSources(object): def __init__(self, **kwargs): self.fs = MultiFS() self.config = kwargs['config'] self.basepath = kwargs['basepath'] def add_folder(self, folder_path, dest=""): real_path = search_path(folder_path, self.basepath, getcwd()) if real_path is None: raise RuntimeError("Couldn't find ansible playbook sources at %s" % folder_path) self.fs.addfs(dest, OSFS(real_path)) # TODO: In the future... # def add_git(self, **kwargs): # def add_file_url(self, url): # Plain method to embed the playbook contents into the cloudformation template def _fs_to_cfninit_plain(self): from base64 import b64encode # Create a metadata object that, when executed by cfn-init, will result in the files # being created in the instance from os.path import join files_obj = {} target = self.config['_inst_playbook_path'] for (fs_dir, fs_dirfiles) in self.fs.walk(): # TODO: more deterministic order of walking through the files if fs_dir[0] == '/': fs_dir = fs_dir[1:] t = join(target, fs_dir) for f in fs_dirfiles: src = join(fs_dir, f) dest = join(t, f) encoding= None try: contents= self.fs.getcontents(src, mode='rb') contents.decode('utf-8') except UnicodeDecodeError: encoding= "base64" contents= b64encode(contents) files_obj[dest] = dict( content= contents, owner= self.config['inst_user'], group= self.config['inst_group'], mode= "000640" ) files_obj[dest].update( {"encoding": encoding} if encoding else {} ) return files_obj def deploy(self, iscm): # Create cfn-init stanzas for creating the added files iscm.iscm_cfninit_add_config({ "files": self._fs_to_cfninit_plain() }, "_ansible_playbooks_install")
def __init__(self, project_fs=None, breakpoint=False): self.project_fs = project_fs self.registry = ElementRegistry() self.libs = {} self.apps = OrderedDict() self.apps_by_lib = defaultdict(list) self.app_settings = defaultdict(SettingsContainer) self.app_system_settings = defaultdict(SettingsContainer) self.cfg = None self.settings = SettingsContainer() self.templates_fs = MultiFS() self.data_fs = MultiFS() self.filesystems = FSContainer({'templates': self.templates_fs, 'data': self.data_fs}) self.filters = FilterContainer(self) self.template_engines = {} self.database_engines = {} self.caches = {} self.mail_servers = {} self.default_mail_server = None self.default_db_engine = None self.debug = False self.auto_reload = False self.known_namespaces = set() self.sites = Sites() self.breakpoint = breakpoint self.suppress_breakpoints = False self.data_tags = defaultdict(list) self.data_tags_by_lib = defaultdict(lambda: defaultdict(list)) # awesome self.preflight = False self.log_signals = False self.debug_echo = False self.log_logger = None self.log_color = True self.log_width = None self.media_urls = None self.media_app = None self.failed_documents = [] self.enum = {} self.enum_by_lib = {} self.signals = Signals() self._moyarc = None self.console = self.create_console()
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir): from fs.multifs import MultiFS from ConfigParser import ConfigParser cfg = ConfigParser() if '#' in fs_path: path, section = fs_path.split('#', 1) else: path = fs_path section = 'fs' cfg.readfp(registry.open(path)) multi_fs = MultiFS() for name, fs_url in cfg.items(section): multi_fs.addfs(name, registry.opendir(fs_url, create_dir=create_dir)) return multi_fs, ''
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir): from fs.multifs import MultiFS from configparser import ConfigParser cfg = ConfigParser() if '#' in fs_path: path, section = fs_path.split('#', 1) else: path = fs_path section = 'fs' cfg.readfp(registry.open(path)) multi_fs = MultiFS() for name, fs_url in cfg.items(section): multi_fs.addfs(name, registry.opendir(fs_url, create_dir=create_dir)) return multi_fs, ''
def get_multi_fs(root): """Create filesystem combining the server (if connected) with profile and scenario containers in blob storage. The priority is in descending order, so the server will be used first if possible :param str root: root directory on server :return: (*fs.base.FS*) -- filesystem instance """ scenario_data = get_blob_fs("scenariodata") profiles = get_blob_fs("profiles") mfs = MultiFS() try: ssh_fs = get_ssh_fs(root) mfs.add_fs("ssh_fs", ssh_fs, write=True, priority=3) except: # noqa print("Could not connect to ssh server") mfs.add_fs("profile_fs", profiles, priority=2) mfs.add_fs("scenario_fs", scenario_data, priority=1) remotes = ",".join([f[0] for f in mfs.iterate_fs()]) print(f"Initialized remote filesystem with {remotes}") return mfs
def test_auto_close(self): """Test MultiFS auto close is working""" multi_fs = MultiFS() m1 = MemoryFS() m2 = MemoryFS() multi_fs.add_fs("m1", m1) multi_fs.add_fs("m2", m2) self.assertFalse(m1.isclosed()) self.assertFalse(m2.isclosed()) multi_fs.close() self.assertTrue(m1.isclosed()) self.assertTrue(m2.isclosed())
def test_no_auto_close(self): """Test MultiFS auto close can be disabled""" multi_fs = MultiFS(auto_close=False) self.assertEqual(repr(multi_fs), "MultiFS(auto_close=False)") m1 = MemoryFS() m2 = MemoryFS() multi_fs.add_fs("m1", m1) multi_fs.add_fs("m2", m2) self.assertFalse(m1.isclosed()) self.assertFalse(m2.isclosed()) multi_fs.close() self.assertFalse(m1.isclosed()) self.assertFalse(m2.isclosed())
def test_multiple_fs_with_use_syspath(self, ctx): testfs = ctx << fs.open_fs('mem://') self.build_fs(testfs, ctx) self.build_zipfs() multi_fs = MultiFS() multi_fs.add_fs('memory', testfs) multi_fs.add_fs('zip', fs.open_fs("zip://test.zip")) env = self.build_env(multi_fs, use_syspath=True) source, path, _ = env.loader.get_source(None, "template_in_zip.j2") self.assertEqual(path, "template_in_zip.j2") os.unlink("test.zip")
def get_scenario_fs(): """Create filesystem combining the server (if connected) with blob storage, prioritizing the server if connected. :return: (*fs.base.FS*) -- filesystem instance """ scenario_data = get_blob_fs("scenariodata") mfs = MultiFS() try: ssh_fs = get_ssh_fs(server_setup.DATA_ROOT_DIR) mfs.add_fs("ssh_fs", ssh_fs, write=True, priority=2) except: # noqa print("Could not connect to ssh server") mfs.add_fs("scenario_fs", scenario_data, priority=1) remotes = ",".join([f[0] for f in mfs.iterate_fs()]) print(f"Initialized remote filesystem with {remotes}") return mfs
def test_multiple_fs(self, ctx): testfs = ctx << fs.open_fs('mem://') self.build_fs(testfs, ctx) self.build_zipfs() multi_fs = MultiFS() multi_fs.add_fs('memory', testfs) multi_fs.add_fs('zip', fs.open_fs("zip://test.zip")) env = self.build_env(multi_fs) template = env.get_template("dir/nested.j2") self.assertEqual(template.render(), "<html>this is a nested template !</html>") template = env.get_template("template_in_zip.j2") self.assertEqual(template.render(), "<html>this template is in a zip</html>") self.assertRaises(jinja2.TemplateNotFound, env.get_template, "other.j2") source, path, _ = env.loader.get_source(None, "template_in_zip.j2") self.assertEqual(path, "template_in_zip.j2")
class Archive(object): _re_element_ref_match = re.compile( r"^(.+\..+)#(.*)$|^(.+)#(.*)$|^#(.+)$", re.UNICODE ).match def __init__( self, project_fs=None, breakpoint=False, strict=False, test_build=False, develop=False, ): self.project_fs = project_fs self.strict = strict self.test_build = test_build self.develop = develop self.registry = ElementRegistry() self.libs = {} self.apps = OrderedDict() self.apps_by_lib = defaultdict(list) self.app_settings = defaultdict(SettingsContainer) self.app_system_settings = defaultdict(SettingsContainer) self.cfg = None self.settings = SettingsContainer() self.templates_fs = MultiFS() self.data_fs = MultiFS() self.filesystems = FSContainer( {"templates": self.templates_fs, "data": self.data_fs} ) self.filters = FilterContainer(self) self.template_engines = {} self.database_engines = {} self.caches = {} self.mail_servers = {} self.default_mail_server = None self.default_db_engine = None self.debug = False self.struct = False self.auto_reload = False self.known_namespaces = set() self.sites = Sites() self.breakpoint = breakpoint self.suppress_breakpoints = False self.data_tags = defaultdict(list) self.data_tags_by_lib = defaultdict(lambda: defaultdict(list)) # awesome self.preflight = False self.log_signals = False self.debug_echo = False self.debug_memory = False self.lib_paths = None self._lib_database = None self.log_logger = None self.log_color = True self.log_width = None self.media_urls = None self.media_app = None self.failed_documents = [] self.enum = {} self.enum_by_lib = {} self.signals = Signals() self._moyarc = None self.console = self.create_console() def __repr__(self): return "<archive>" @property def media_url(self): return self.media_urls[0] if self.media_urls else None @property def moyarc(self): if self._moyarc is None: try: with io.open(os.path.expanduser("~/.moyarc"), "rt") as f: self._moyarc = settings.SettingsContainer.read_from_file(f) except IOError: self._moyarc = settings.SettingsContainer() return self._moyarc @property def lib_database(self): if self._lib_database is None: start = time() self._lib_database = self.scan_libs(self.lib_paths, self.project_fs) log.debug( "%s scanned %i libs %0.1fms", self, len(self._lib_database), (time() - start) * 1000.0, ) return self._lib_database def find_lib(self, version_spec): options = [] _version_spec = versioning.VersionSpec(version_spec) for name, version, path, dir_name in self.lib_database: if _version_spec.name != name: continue if _version_spec.comparisons and not _version_spec.compare(version): continue if "://" in path: base_fs = open_fs(path) else: base_fs = self.project_fs.opendir(path) lib_fs = base_fs.opendir(dir_name) return lib_fs def get_relative_path(self, path): """Get a relative path from the project base""" base = self.project_fs.getsyspath("/", allow_none=True) if base is None: return path return relativefrom(base, path) def open_fs(self, fs_url, create=False): if isinstance(fs_url, text_type): if "://" in fs_url: fs = open_fs(fs_url, create=create) else: if create: self.project_fs.makedirs(fs_url, recreate=True) fs = self.project_fs.opendir(fs_url) else: fs = self.project_fs.opendir(fs_url) else: fs = fs_url return fs def get_console_file(self): if self.log_logger: console_file = logtools.LoggerFile(self.log_logger) else: console_file = None return console_file def create_console(self): console = Console( out=self.get_console_file(), nocolors=not ( self.log_color and self.moyarc.get_bool("console", "color", True) ), width=self.log_width or None, ) return console @classmethod def scan_libs(cls, lib_paths, base_fs): """Read libs from paths.""" libs = [] for path in lib_paths: try: if "://" in path: libs_fs = open_fs(path) else: libs_fs = base_fs.opendir(path) except FSError as error: startup_log.warning("unable to read from '%s' (%s)", path, error) continue for resource in libs_fs.filterdir("/", exclude_files=["*"]): with libs_fs.opendir(resource.name) as lib_fs: try: with lib_fs.open("lib.ini", "rb") as ini_file: lib_settings = SettingsContainer.read_from_file(ini_file) except ResourceNotFound: continue name = lib_settings.get("lib", "name", None) if name is None: continue version = lib_settings.get("lib", "version", None) if version is None: continue libs.append((name, version, path, resource.name)) return libs def build_libs(self, ignore_errors=False): if self.test_build: ignore_errors = True libs = [lib for lib in itervalues(self.libs) if not lib.built] if not libs: return start = time() self.build([doc for lib in libs for doc in lib.documents], log_time=False) for lib in libs: lib.finalize(ignore_errors=ignore_errors) startup_log.debug("%s built libraries %0.1fms", self, (time() - start) * 1000.0) def build(self, documents, context=None, log_time=True, fs=None): """Build all documents in the library""" # This handles tags defined out of order if fs is not None: self.project_fs = fs start = time() if isinstance(documents, Document): documents = [documents] else: documents = list(documents) build_queue = deque() for doc in documents: if not doc.document_finalized and doc.structure: build_queue.append((None, [doc.structure.root_node])) unbuildable = set() unbuildable_clear = unbuildable.clear unbuilt = [] if context is None: context = Context() context_root = context.root get_doc_id = attrgetter("doc_id") while build_queue: parent_element, nodes = build_queue[0] if nodes: node = nodes.pop() context_root["_lib_long_name"] = node.lib_long_name element = node.build(self, context) if element: unbuildable_clear() build_queue.appendleft( (element, sorted(node.children, key=get_doc_id, reverse=True)) ) else: if element is not None: if node in unbuildable: # A previously deferred node, but we still can't built it build_queue.popleft() # Can't process siblings either unbuilt.append(node) else: # Defer this node till later nodes.append(node) build_queue.rotate(-1) unbuildable.add(node) else: if parent_element is not None: unbuildable_clear() try: parent_element.finalize(context) except Exception as e: raise failed_doc = FailedDocument( path=node.structure.document.location, code=node.structure.xml, line=node.source_line, col=None, msg=text_type(e), ) self.failed_documents.append(failed_doc) build_queue.popleft() if unbuilt: for node in unbuilt: nearest = nearest_word( node.tag_name, self.registry.get_elements_in_xmlns(node.xmlns) ) msg = "unknown tag {} in {}".format( node.tag_display_name, node.structure ) diagnosis = None if nearest: diagnosis = "did you mean <{}>?".format(nearest) else: find_xmlns = self.registry.find_xmlns(node.tag_name) if find_xmlns: diagnosis = "did you mean <{}> in XMLNS '{}'?".format( node.tag_name, find_xmlns ) failed_doc = FailedDocument( path=node.structure.document.location, code=node.structure.xml, line=node.source_line, col=None, msg=msg, diagnosis=diagnosis, ) self.failed_documents.append(failed_doc) return False if self.strict: failed = 0 for doc in documents: failed += self.check_attributes(doc) if failed: startup_log.debug("%s %s strict check(s) failed", self, failed) else: startup_log.debug("%s strict checks passed", self) for doc in documents: doc.document_finalize(context) if log_time: doc_text = ", ".join(text_type(doc) for doc in documents) startup_log.debug("%s built %0.1fms", doc_text, (time() - start) * 1000.0) return True def check_attributes(self, doc): failed = 0 for element_name, element in doc.elements.items(): for k, attribute in element._tag_attributes.items(): if k in element._attrs: attr_text = element._attrs[k] error = attribute.type.check(attr_text) if error: failed += 1 failed_doc = FailedDocument( path=doc.location, code=element._code, line=element.source_line or 0, col=None, msg="error in parameter '{}'; {}".format(k, error), diagnosis="This check is performed when [project]/strict is enabled, or with 'moya runserver --strict' switch", ) self.failed_documents.append(failed_doc) return failed def populate_context(self, context): from .context.expressiontime import ExpressionDateTime root = context.root root["libs"] = self.libs root["apps"] = self.apps root["filters"] = self.filters root["debug"] = self.debug root["develop"] = self.develop root["fs"] = self.get_context_filesystems() root["now"] = ExpressionDateTime.utcnow() from . import __version__ root["moya"] = {"version": __version__} root["enum"] = self.enum root["media_url"] = self.media_url root["secret"] = self.secret context.set_dynamic( ".app", lambda context: getattr(context.get(".call", None), "app") ) @classmethod def get_callable_from_document( cls, path, element_ref, breakpoint=False, fs="./", default_context=False, archive=None, lib=None, ): """Shortcut that imports a single document and returns a callable""" if archive is None: archive = cls() if lib is None: lib = archive.create_library(long_name="moya.run", namespace=namespaces.run) lib.import_document(fs, path) archive.build_libs() app, element = lib.documents[0].get_element(element_ref) if element is None: raise errors.ElementNotFoundError(element_ref, app=app) call = CallableElement(archive, element, app, breakpoint=breakpoint) if default_context: def do_call(*args, **kwargs): c = Context() c["console"] = Console() return call(c, *args, **kwargs) do_call.archive = archive return do_call return call @classmethod def parse_element_ref(cls, s, cache={}): try: return cache[s] except KeyError: match = cls._re_element_ref_match(s) if match is None: result = None, None, s else: libname, lib_elementname, appname, app_elementname, docname = ( match.groups() ) result = appname, libname, app_elementname or lib_elementname or docname cache[s] = result return result def get_library(self, library_name): """Get a library from either its short name, or its long name""" try: return self.libs[library_name] except KeyError: raise errors.UnknownLibraryError(lib=library_name) def has_library(self, library_name): return library_name in self.libs def load_library_from_module(self, py, **kwargs): __import__(py) module = sys.modules[py] location = os.path.dirname(os.path.abspath(module.__file__)) import_fs = open_fs(location) lib = self.load_library(import_fs, **kwargs) return lib def load_library( self, import_fs, priority=None, template_priority=None, data_priority=None, long_name=None, rebuild=False, ): """Load a new library in to the archive""" lib = self.create_library(import_fs, long_name=long_name, rebuild=rebuild) if priority is not None: lib.priority = priority lib.data_priority = data_priority if lib.templates_info: fs_url = lib.templates_info["location"] if template_priority is None: try: template_priority = int(lib.templates_info.get("priority", "0")) except ValueError: startup_log.error( "{} invalid value for [templates]/priority, assuming 0".format( lib ) ) template_priority = 0 lib.template_priority = template_priority if "://" in fs_url: fs = open_fs(fs_url) else: fs = import_fs.opendir(fs_url) self.templates_fs.add_fs(lib.long_name, fs, priority=template_priority) return lib def get_or_create_library(self, long_name, import_fs=None): """Get a library, or create it if it doesn't exists""" if long_name is not None and long_name in self.libs: return self.libs[long_name] return self.create_library(import_fs) def create_library( self, import_fs=None, long_name=None, namespace=None, rebuild=False ): """Create a new library, and import documents""" lib = Library( self, import_fs, long_name=long_name, namespace=namespace, rebuild=rebuild ) self.libs[lib.long_name] = lib return lib def finalize(self, ignore_errors=False): self.build_libs(ignore_errors=ignore_errors) for lib in itervalues(self.libs): lib.on_archive_finalize() if self.database_engines: for app in itervalues(self.apps): for model in app.lib.get_elements_by_type((namespaces.db, "model")): if not model.abstract: model.get_table_and_class(app) def create_app(self, name, lib_name): if name in self.apps: raise errors.ArchiveError( "Application name '{}' was previously installed with {}".format( name, self.apps[name].lib ) ) app = Application(self, name, lib_name) self.apps[name] = app app.settings.update(self.app_settings[name]) app.system_settings.update(self.app_system_settings[name]) self.apps_by_lib[lib_name].append(name) return app def get_app(self, app_id): try: return self.apps[app_id] except KeyError: return None def has_app(self, app_id): return app_id in self.apps def get_app_from_lib(self, lib, current=None): if isinstance(lib, Library): lib_name = lib.long_name else: lib_name = text_type(lib) if "." not in lib_name: return lib_name apps = self.apps_by_lib[lib_name] if len(apps) != 1: if not apps: raise errors.AppRequiredError(lib_name) if current: if current.name in apps: return current raise errors.AmbiguousAppError(lib_name, apps) return self.apps[apps[0]] def get_app_from_lib_default(self, lib, default=None): if isinstance(lib, Library): lib_name = lib.long_name else: lib_name = text_type(lib) if "." not in lib_name: return lib_name apps = self.apps_by_lib[lib_name] if len(apps) != 1: return default return self.apps[apps[0]] def find_app(self, name): """ Find an app from either its name or its lib name. If a lib name is supplied and there are more than one app, an AmbiguousAppError is raise """ if isinstance(name, Application): return name if not name: raise errors.UnknownAppError( "Value {} is not a valid app or lib name".format( to_expression(None, name) ) ) name = text_type(name) try: if "." in name: apps = self.apps_by_lib[name] if not apps: raise KeyError("No app called '{}'".format(name)) if len(apps) != 1: raise errors.AmbiguousAppError(name, apps) return self.apps[apps[0]] else: return self.apps[name] except KeyError: raise errors.UnknownAppError(app=name) def find_app_default(self, name, default=None): try: return self.find_app(name) except errors.UnknownAppError: return default def get_app_settings(self, name): """ Get settings object for an application. """ app = self.find_app(name) return app.settings def detect_app(self, context, name): """ Find an app from either its name or its libname if the app is ambiguous, attempt to detect it from the callstack. """ if isinstance(name, Application): return name if not name: raise errors.UnknownAppError( "Value {} is not a valid app or lib name".format(context.expr(name)) ) try: if "." in name: apps = self.apps_by_lib[name] if not apps: raise KeyError("No app called '{}'".format(name)) if len(apps) != 1: _app = context[".app"] if _app and _app.name in apps: return _app for c in reversed(context["._callstack"]): if c.app.name in apps: return c.app raise errors.AmbiguousAppError(name, apps) return self.apps[apps[0]] else: return self.apps[name] except KeyError: raise errors.UnknownAppError(app=name) def get_lib(self, name): if isinstance(name, Library): return name.long_name if "." in name: return self.libs[name].long_name return self.find_app(name).lib.long_name def add_data_tag(self, element_type, tag): self.data_tags[element_type].append(tag) self.data_tags_by_lib[tag.lib.long_name][element_type].append(tag) def get_data(self, context, namespace, tag_name): """Get data from a data tag""" tag_type = (namespace, tag_name) return [ e.get_all_data_parameters(context) for e in self.data_tags.get(tag_type, []) if e.check(context) ] def get_data_item(self, context, namespace, tag_name, filter_map, lib=None): """Get data from a data tag""" tag_type = (namespace, tag_name) for e in self.data_tags.get(tag_type, []): if lib is not None: if e.lib != lib: continue data = e.get_all_data_parameters(context) if all( filter_map[k] == data.get(k, Ellipsis) for k, v in filter_map.items() ): return data def get_data_from_element(self, context, element): return element.get_all_data_parameters(context) def get_data_elements(self, context, namespace, tag_name): """Get data from a data tag""" tag_type = (namespace, tag_name) return [ DataElement(e, context) for e in self.data_tags.get(tag_type, []) if e.check(context) ] def get_app_data_elements(self, context, namespace, tag_name): data = [] tag_type = (namespace, tag_name) for app in itervalues(self.apps): _elements = [] append = _elements.append for e in self.data_tags_by_lib[app.lib.long_name].get(tag_type, []): if e.check(context): append(DataElement(e, context)) if _elements: data.append((app, _elements)) return data def add_filesystem(self, name, fs, create=False): try: add_fs = self.open_fs(fs, create=create) except FSError as e: raise errors.StartupFailedError( "unable to open filesystem '{name}' ({e})".format( name=name, e=text_type(e) ) ) self.filesystems[name] = add_fs # startup_log.debug("%s fs added as '%s'", add_fs, name) startup_log.debug( "<%s '%s'> filesystem added", type(add_fs).__name__.lower(), name ) return fs def get_filesystem(self, name): return self.filesystems[name] def has_filesystem(self, name): return name in self.filesystems def lookup_filesystem(self, element, name): try: return self.filesystems[name] except KeyError: raise element.throw( "fs.no-filesystem", "no filesystem called '{0}'".format(name), diagnosis="You can view installed filesystems from the command line with **moya fs**", ) def get_reader(self, name="data"): fs = self.get_filesystem(name) return DataReader(fs) def get_context_filesystems(self): return FSContainer((k, FSWrapper(fs)) for k, fs in iteritems(self.filesystems)) def get_translations(self, app_or_lib, languages): return self.find_app(app_or_lib).lib.translations.get(languages) def init_template_engine(self, system, settings): if system in self.template_engines: return engine = TemplateEngine.create(system, self, self.templates_fs, settings) self.template_engines[system] = engine startup_log.debug("%s template engine initialized", engine) def init_cache(self, name, settings): cache = Cache.create(name, settings) self.caches[name] = cache startup_log.debug("%s cache added", cache) def has_cache(self, name): """Check if a cache is present and enabled""" if name not in self.caches: return False cache = self.caches[name] return cache.enabled def get_cache(self, name): if name in self.caches: return self.caches[name] cache = self.caches[name] = Cache.create( "runtime", SettingsSectionContainer({"type": "dict"}) ) return cache def get_mailserver(self, name=None): name = name or self.default_mail_server or "default" try: return self.mail_servers[name or "default"] except KeyError: raise errors.MoyaException( "email.no-server", "no email server called '{0}'".format(name) ) def init_templates(self, name, location, priority): templates_fs = self.filesystems.get("templates") fs = self.open_fs(location) templates_fs.add_fs(name, fs, priority=priority) # startup_log.debug("added templates filesystem, priority %s", priority) def get_template_engine(self, engine="moya"): return self.template_engines[engine] def get_default_template_engine(self, app): engine = app.lib.templates_info.get("default_engine", "moya") return engine def init_settings(self, cfg=None): cfg = cfg or self.cfg self.secret = cfg.get("project", "secret", "") self.preflight = cfg.get_bool("project", "preflight", False) self.debug = cfg.get_bool("project", "debug") self.strict = self.strict or cfg.get_bool("project", "strict") self.develop = self.develop or cfg.get_bool("project", "develop") self.log_signals = cfg.get_bool("project", "log_signals") self.debug_echo = cfg.get_bool("project", "debug_echo") self.debug_memory = cfg.get_bool("project", "debug_memory") self.lib_paths = cfg.get_list("project", "paths", "./local\n./external") self.lib_paths = self.lib_paths[:] + [MOYA_LIBS_PATH] if "console" in cfg: self.log_logger = cfg.get("console", "logger", None) self.log_color = cfg.get_bool("console", "color", True) self.log_width = cfg.get_int("console", "width", None) self.console = self.create_console() self.sites.set_defaults(cfg["site"]) if "templates" not in self.caches: self.caches["templates"] = Cache.create( "templates", SettingsSectionContainer({"type": "dict"}) ) if "runtime" not in self.caches: self.caches["runtime"] = Cache.create( "runtime", SettingsSectionContainer({"type": "dict"}) ) require_name = ["app", "smtp", "db"] self.auto_reload = cfg.get_bool("autoreload", "enabled") if self.strict: startup_log.debug("strict mode is enabled") if self.develop: startup_log.debug("develop mode is enabled") for section_name, section in iteritems(cfg): section = SectionWrapper(section_name, section) if ":" in section_name: what, name = section_name.split(":", 1) else: what = section_name name = None if what in require_name and not name: raise errors.StartupFailedError( "name required in section, [{section}:?]".format(section=what) ) if what in ("project", "debug", "autoreload", "console", "customize", ""): continue if what == "settings": if name is None: self.settings.update( (k, SettingContainer(v)) for k, v in iteritems(section) ) else: self.app_settings[name].update( (k, SettingContainer(v)) for k, v in iteritems(section) ) elif what == "application": self.app_system_settings[name].update(section) elif what == "lib": if self.has_library(name): lib = self.get_library(name) lib.settings.update( (k, SettingContainer(v)) for k, v in iteritems(section) ) elif what == "fs": location = section.get("location") if not location: raise errors.StartupFailedError( "a value for 'location' is required in [{}]".format( section_name ) ) create = section.get_bool("create", False) self.add_filesystem(name, location, create=create) elif what == "data": location = section.get("location") data_fs = self.open_fs(location) self.data_fs.add_fs( "archive", data_fs, priority=section.get_int("priority", 0) ) elif what == "cache": self.init_cache(name, section) elif what == "templates": location = section["location"] try: priority = int(section["priority"]) except (IndexError, ValueError): priority = 0 self.init_templates(name, location, priority) elif what == "db": from .db import add_engine add_engine(self, name, section) elif what == "media": priority = section.get_int("priority", 1) location = section["location"] static_media_fs = self.open_fs(location) media_fs = MultiFS() media_fs.add_fs("static", static_media_fs, priority=priority) self.add_filesystem("media", media_fs) self.media_urls = section.get_list("url") self.media_app = section.get("app", "media") elif what == "smtp": host = section["host"] port = section.get_int("port", 25) timeout = section.get_int("timeout", None) username = section.get("username", None) password = section.get("password", None) default = section.get_bool("default", False) sender = section.get("sender", None) server = MailServer( host, name=name, port=port, default=default, timeout=timeout, username=username, password=password, sender=sender, ) self.mail_servers[name] = server if self.default_mail_server is None or default: self.default_mail_server = name if default: startup_log.debug("%r (default) created", server) else: startup_log.debug("%r created", server) elif what == "site": if name: self.sites.add_from_section(name, section) elif what == "themes": location = section["location"] theme_fs = self.open_fs(location) self.add_filesystem("themes", theme_fs) # startup_log.debug("added theme filesystem '%s'", location) else: startup_log.warn("unknown settings section, [%s]", section_name) self.init_template_engine("moya", {}) def init_media(self): if "media" not in self.filesystems: return if not self.media_urls: if not self.media_app: raise errors.StartupFailedError( "no 'url' or 'app' specified in [media] section" ) if self.media_app not in self.apps: startup_log.warning( "app set in [media]/app has not been installed ({})".format( self.media_app ) ) return try: self.media_urls = [self.apps[self.media_app].mounts[0][1]] except: raise errors.StartupFailedError( "unable to detect media url! (specify in [media]/url)" ) for i, _url in enumerate(self.media_urls): startup_log.debug("media url #%s is %s", i, _url) media_fs = self.filesystems["media"] media_mount_fs = MountFS() for app in itervalues(self.apps): for media_name, media_sub_fs in iteritems(app.lib.media): name = "%s_%s" % (app.name, media_name) media_path = "%s-%s" % (app.name, media_name) app.media[media_name] = media_path if name in self.filesystems: mount_media = self.filesystems[name] else: mount_media = media_sub_fs if name not in self.filesystems: self.filesystems[name] = mount_media media_mount_fs.mount(media_path, mount_media) media_fs.add_fs("media", media_mount_fs) def init_data(self): data_fs = self.data_fs for lib in itervalues(self.libs): data_priority = lib.data_info.get("priority", 0) if lib.data_priority is not None: data_priority = lib.data_priority if lib.data_fs is not None: data_fs.add_fs(lib.long_name, lib.data_fs, priority=data_priority) def get_media_url(self, context, app, media, path="", url_index=None): """Get a URL to media in a given app""" if url_index is None: url_index = context.inc("._media_url_index") if not self.media_urls: return None url_no = url_index % len(self.media_urls) if app is None: return url_join(self.media_urls[url_no] or "", path) return url_join( self.media_urls[url_no] or "", app.get_media_directory(media), path ) @property def is_media_enabled(self): """Check if the media system is enabled""" return bool(self.media_urls) def get_element(self, element_ref, app=None, lib=None): """Gets an element from a reference""" app_id, lib_id, name = self.parse_element_ref(element_ref) if lib_id: lib = self.get_library(lib_id) element_app = self.get_app_from_lib_default(lib) element = lib.get_named_element(name) elif app_id: try: element_app = self.apps[app_id] except KeyError: raise errors.ElementNotFoundError(element_ref, app=app, lib=lib) element = element_app.lib.get_named_element(name) else: if app is not None: element_app = app element = app.lib.get_named_element(name) elif lib is not None: element_app = app element = lib.get_named_element(name) else: raise errors.ElementNotFoundError(element_ref, app=app, lib=lib) if element is None: raise errors.ElementNotFoundError(element_ref, app=app, lib=lib) return FoundElement(element_app, element) def get_app_element(self, element_ref): element_app = self.find_app(element_ref.split("#", 1)[0]) app, element = self.get_element(element_ref, app=element_app) return app or element_app, element def resolve_template_path(self, path, app_name, base_path="/"): """Get a template path in the appropriate directory for an app""" if path.startswith("/"): return path if isinstance(app_name, Application): app = app_name else: app = self.find_app(text_type(app_name)) app = app or self.detect_app() template_path = abspath(join(base_path, app.templates_directory, path)) return template_path def get_template_lib(self, path, _cache=LRUCache()): if path in _cache: return _cache[path] lib = None for app in itervalues(self.apps): if path.startswith(app.templates_directory): lib = _cache[path] = app.lib.long_name break return lib def get_elements_by_type(self, ns, type): """Get all elements of a given namespace, type""" _elements = [] extend = _elements.extend element_type = (ns, type) for lib in itervalues(self.libs): extend(lib.get_elements_by_type(element_type)) return _elements def add_enum(self, libid, enum): """Add an enumeration""" self.enum[libid] = enum libname = libid.partition("#")[0] if enum.name: self.enum_by_lib.setdefault(libname, {})[enum.name] = enum def get_enum(self, enum_libid): """Get an enumeration""" return self.enum[enum_libid] def get_lib_enums(self, libname): """Get enumeration for a given lib""" return self.enum_by_lib.get(libname, None) or {} def fire(self, context, signal_name, app=None, sender=None, data=None): """Fire a signal""" if data is None: data = {} signal_obj = {"name": signal_name, "app": app, "sender": sender, "data": data} if self.log_signals: _params = context.to_expr(data) if sender: signal_log.debug( 'firing "%s" from "%s" %s', signal_name, sender, _params ) else: signal_log.debug('firing "%s" %s', signal_name, _params) for element_ref in self.signals.filter_handlers(signal_name, sender): _, libname, _ = self.parse_element_ref(element_ref) for app_name in self.apps_by_lib[libname]: app = self.apps[app_name] try: _callable = self.get_callable(element_ref, app=app) _callable(context, signal=signal_obj) except errors.LogicError as e: # We can't risk any unhandled exceptions here try: log.error("%s unhandled in signal '%s'", e, signal_name) except: pass try: if context[".debug"]: context[".console"].obj(context, e) except: pass def get_callable(self, element_ref, app=None, breakpoint=False): ref_app, element = self.get_element(element_ref, app=app) if element is None: raise errors.ElementNotFoundError(element_ref) return CallableElement(self, element, ref_app or app, breakpoint=breakpoint) def get_callable_from_element(self, element, app=None, breakpoint=False): return CallableElement(self, element, app, breakpoint=breakpoint) def call(self, element_ref, context, app, *args, **kwargs): _callable = self.get_callable(element_ref, app=app) return _callable(context, *args, **kwargs) def call_params(self, element_ref, context, app, params): _callable = self.get_callable(element_ref, app=app) return _callable(context, **params) def debug_call(self, element_ref, context, app, *args, **kwargs): _callable = self.get_callable(element_ref, app=app, breakpoint=True) return _callable(context, *args, **kwargs) __call__ = call
class Archive(object): _re_element_ref_match = re.compile(r'^(.+\..+)#(.*)$|^(.+)#(.*)$|^#(.+)$', re.UNICODE).match def __init__(self, project_fs=None, breakpoint=False): self.project_fs = project_fs self.registry = ElementRegistry() self.libs = {} self.apps = OrderedDict() self.apps_by_lib = defaultdict(list) self.app_settings = defaultdict(SettingsContainer) self.app_system_settings = defaultdict(SettingsContainer) self.cfg = None self.settings = SettingsContainer() self.templates_fs = MultiFS() self.data_fs = MultiFS() self.filesystems = FSContainer({'templates': self.templates_fs, 'data': self.data_fs}) self.filters = FilterContainer(self) self.template_engines = {} self.database_engines = {} self.caches = {} self.mail_servers = {} self.default_mail_server = None self.default_db_engine = None self.debug = False self.auto_reload = False self.known_namespaces = set() self.sites = Sites() self.breakpoint = breakpoint self.suppress_breakpoints = False self.data_tags = defaultdict(list) self.data_tags_by_lib = defaultdict(lambda: defaultdict(list)) # awesome self.preflight = False self.log_signals = False self.debug_echo = False self.log_logger = None self.log_color = True self.log_width = None self.media_urls = None self.media_app = None self.failed_documents = [] self.enum = {} self.signals = Signals() self._moyarc = None self.console = self.create_console() def __repr__(self): return "<archive>" @property def media_url(self): return self.media_urls[0] if self.media_urls else None @property def moyarc(self): if self._moyarc is None: try: with io.open(os.path.expanduser("~/.moyarc"), 'rt') as f: self._moyarc = settings.SettingsContainer.read_from_file(f) except IOError: self._moyarc = settings.SettingsContainer() return self._moyarc def open_fs(self, fs_url): if isinstance(fs_url, text_type): if '://' in fs_url: fs = fsopendir(fs_url) else: fs = self.project_fs.opendir(fs_url) else: fs = fs_url return fs def get_console_file(self): if self.log_logger: console_file = logtools.LoggerFile(self.log_logger) else: console_file = None return console_file def create_console(self): console = Console(out=self.get_console_file(), nocolors=not (self.log_color and self.moyarc.get_bool('console', 'color', True)), width=self.log_width or None) return console def build_libs(self, ignore_errors=False): libs = [lib for lib in itervalues(self.libs) if not lib.built] if not libs: return start = time() self.build([doc for lib in libs for doc in lib.documents], log_time=False) for lib in libs: lib.finalize(ignore_errors=ignore_errors) startup_log.debug("%s built libraries %0.1fms", self, (time() - start) * 1000.0) def build(self, documents, context=None, log_time=True, fs=None): """Build all documents in the library""" # This handles tags defined out of order if fs is not None: self.project_fs = fs start = time() if isinstance(documents, Document): documents = [documents] else: documents = list(documents) build_queue = deque() build_queue_appendleft = build_queue.appendleft for doc in documents: if not doc.document_finalized and doc.structure: build_queue.append(_BuildFrame([doc.structure.root_node])) unbuildable = set() unbuildable_clear = unbuildable.clear unbuilt = [] if context is None: context = Context() context_root = context.root get_doc_id = attrgetter('doc_id') while build_queue: nodes = build_queue[0] if nodes: node = nodes.pop() context_root['_lib_long_name'] = node.lib_long_name element = node.build(self, context) if element: unbuildable_clear() build_queue_appendleft(_BuildFrame(sorted(node.children, key=get_doc_id, reverse=True), element=element)) else: if element is not None: if node in unbuildable: # A previously deferred node, but we still can't built it build_queue.popleft() # Can't process siblings either unbuilt.append(node) else: # Defer this node till later nodes.append(node) build_queue.rotate(-1) unbuildable.add(node) else: if nodes.element is not None: unbuildable_clear() try: nodes.element.finalize(context) except Exception as e: raise failed_doc = FailedDocument(path=node.structure.document.location, code=node.structure.xml, line=node.source_line, col=None, msg=text_type(e)) self.failed_documents.append(failed_doc) #return False build_queue.popleft() if unbuilt: for node in unbuilt: nearest = nearest_word(node.tag_name, self.registry.get_elements_in_xmlns(node.xmlns)) msg = "unknown tag {} in {}".format(node.tag_display_name, node.structure) diagnosis = None if nearest: diagnosis = "did you mean <{}>?".format(nearest) else: find_xmlns = self.registry.find_xmlns(node.tag_name) if find_xmlns: diagnosis = "did you mean <{}> in XMLNS '{}'?".format(node.tag_name, find_xmlns) failed_doc = FailedDocument(path=node.structure.document.location, code=node.structure.xml, line=node.source_line, col=None, msg=msg, diagnosis=diagnosis) self.failed_documents.append(failed_doc) return False for doc in documents: doc.document_finalize(context) if log_time: doc_text = ', '.join(text_type(doc) for doc in documents) startup_log.debug("%s built %0.1fms", doc_text, (time() - start) * 1000.0) return True def populate_context(self, context): from .context.expressiontime import ExpressionDateTime root = context.root root['libs'] = self.libs root['apps'] = self.apps root['filters'] = self.filters root['debug'] = self.debug root['develop'] = self.develop root['fs'] = self.get_context_filesystems() root['now'] = ExpressionDateTime.utcnow() from . import __version__ root['moya'] = {'version': __version__} root['enum'] = self.enum root['media_url'] = self.media_url root['secret'] = self.secret context.set_dynamic('.app', lambda context: getattr(context.get('.call', None), 'app')) @classmethod def get_callable_from_document(cls, path, element_ref, breakpoint=False, fs='./', default_context=False, archive=None, lib=None): """Shortcut that imports a single document and returns a callable""" if archive is None: archive = cls() if lib is None: lib = archive.create_library(long_name="moya.run", namespace=namespaces.run) lib.import_document(fs, path) archive.build_libs() app, element = lib.documents[0].get_element(element_ref) if element is None: raise errors.ElementNotFoundError(element_ref, app=app) call = CallableElement(archive, element, app, breakpoint=breakpoint) if default_context: def do_call(*args, **kwargs): c = Context() c['console'] = Console() return call(c, *args, **kwargs) do_call.archive = archive return do_call return call @classmethod def parse_element_ref(cls, s, cache={}): try: return cache[s] except KeyError: match = cls._re_element_ref_match(s) if match is None: result = None, None, s else: libname, lib_elementname, appname, app_elementname, docname = match.groups() result = appname, libname, app_elementname or lib_elementname or docname cache[s] = result return result def get_library(self, library_name): """Get a library from either its short name, or its long name""" try: return self.libs[library_name] except KeyError: raise errors.UnknownLibraryError(lib=library_name) def has_library(self, library_name): return library_name in self.libs def load_library(self, import_fs, priority=None, template_priority=None, long_name=None, rebuild=False): """Load a new library in to the archive""" lib = self.create_library(import_fs, long_name=long_name, rebuild=rebuild) if priority is not None: lib.priority = priority if lib.templates_info: fs_url = lib.templates_info['location'] if template_priority is None: try: template_priority = int(lib.templates_info.get('priority', '0')) except ValueError: startup_log.error("{} Invalid value for [templates]/priority, assuming 0".format(lib)) template_priority = 0 lib.template_priority = template_priority if '://' in fs_url: fs = fsopendir(fs_url) else: fs = import_fs.opendir(fs_url) self.templates_fs.addfs(lib.long_name, fs, priority=template_priority) return lib def get_or_create_library(self, long_name, import_fs=None): """Get a library, or create it if it doesn't exists""" if long_name is not None and long_name in self.libs: return self.libs[long_name] return self.create_library(import_fs) def create_library(self, import_fs=None, long_name=None, namespace=None, rebuild=False): """Create a new library, and import documents""" lib = Library(self, import_fs, long_name=long_name, namespace=namespace, rebuild=rebuild) self.libs[lib.long_name] = lib return lib def finalize(self, ignore_errors=False): self.build_libs(ignore_errors=ignore_errors) for lib in itervalues(self.libs): lib.on_archive_finalize() if self.database_engines: for app in itervalues(self.apps): for model in app.lib.get_elements_by_type((namespaces.db, "model")): if not model.abstract: model.get_table_and_class(app) def create_app(self, name, lib_name): if name in self.apps: raise errors.ArchiveError("Application name '{}' was previously installed with {}".format(name, self.apps[name].lib)) app = Application(self, name, lib_name) self.apps[name] = app app.settings.update(self.app_settings[name]) app.system_settings.update(self.app_system_settings[name]) self.apps_by_lib[lib_name].append(name) return app def get_app(self, app_id): try: return self.apps[app_id] except IndexError: return None def has_app(self, app_id): return app_id in self.apps def get_app_from_lib(self, lib): if isinstance(lib, Library): lib_name = lib.long_name else: lib_name = text_type(lib) if '.' not in lib_name: return lib_name apps = self.apps_by_lib[lib_name] if len(apps) != 1: if not apps: raise errors.AppRequiredError(lib_name) raise errors.AmbiguousAppError(lib_name, apps) return self.apps[apps[0]] def get_app_from_lib_default(self, lib, default=None): if isinstance(lib, Library): lib_name = lib.long_name else: lib_name = text_type(lib) if '.' not in lib_name: return lib_name apps = self.apps_by_lib[lib_name] if len(apps) != 1: return default return self.apps[apps[0]] def find_app(self, name): """Find an app from either its name or its lib name. If a lib name is supplied and there are more than one app, an AmbiguousAppError is raise """ if isinstance(name, Application): return name if not name: raise errors.UnknownAppError("Value {} is not a valid app or lib name".format(to_expression(None, name))) name = text_type(name) try: if '.' in name: apps = self.apps_by_lib[name] if not apps: raise KeyError("No app called '{}'".format(name)) if len(apps) != 1: raise errors.AmbiguousAppError(name, apps) return self.apps[apps[0]] else: return self.apps[name] except KeyError: raise errors.UnknownAppError(app=name) def get_lib(self, name): if isinstance(name, Library): return name.long_name if '.' in name: return self.libs[name].long_name return self.find_app(name).lib.long_name def add_data_tag(self, element_type, tag): self.data_tags[element_type].append(tag) self.data_tags_by_lib[tag.lib.long_name][element_type].append(tag) def get_data(self, context, namespace, tag_name): """Get data from a data tag""" tag_type = (namespace, tag_name) return [e.get_all_data_parameters(context) for e in self.data_tags.get(tag_type, []) if e.check(context)] def get_data_item(self, context, namespace, tag_name, filter_map): """Get data from a data tag""" tag_type = (namespace, tag_name) for e in self.data_tags.get(tag_type, []): data = e.get_all_data_parameters(context) if all(filter_map[k] == data.get(k, Ellipsis) for k, v in filter_map.items()): return data def get_data_from_element(self, context, element): return element.get_all_data_parameters(context) def get_data_elements(self, context, namespace, tag_name): """Get data from a data tag""" tag_type = (namespace, tag_name) return [DataElement(e, context) for e in self.data_tags.get(tag_type, []) if e.check(context)] def get_app_data_elements(self, context, namespace, tag_name): data = [] tag_type = (namespace, tag_name) for app in itervalues(self.apps): _elements = [] append = _elements.append for e in self.data_tags_by_lib[app.lib.long_name].get(tag_type, []): if e.check(context): append(DataElement(e, context)) if _elements: data.append((app, _elements)) return data def add_filesystem(self, name, fs): add_fs = self.open_fs(fs) self.filesystems[name] = add_fs startup_log.debug("%s fs added as '%s'", add_fs, name) return fs def get_filesystem(self, name): return self.filesystems[name] def get_reader(self, name="data"): fs = self.get_filesystem(name) return DataReader(fs) def get_context_filesystems(self): return FSContainer((k, FSWrapper(fs)) for k, fs in iteritems(self.filesystems)) def get_translations(self, app_or_lib, languages): return self.find_app(app_or_lib).lib.translations.get(languages) def init_template_engine(self, system, settings): if system in self.template_engines: return engine = TemplateEngine.create(system, self, self.templates_fs, settings) self.template_engines[system] = engine startup_log.debug('%s template engine initialized', engine) def init_cache(self, name, settings): cache = Cache.create(name, settings) self.caches[name] = cache startup_log.debug('%s cache added', cache) def get_cache(self, name): if name in self.caches: return self.caches[name] cache = self.caches[name] = Cache.create('runtime', SettingsSectionContainer({'type': 'dict'})) return cache def get_mailserver(self, name=None): name = name or self.default_mail_server or 'default' try: return self.mail_servers[name or 'default'] except KeyError: raise errors.MoyaException("email.no-server", "no email server called '{0}'".format(name)) def init_templates(self, name, location, priority): templates_fs = self.filesystems.get("templates") fs = self.open_fs(location) templates_fs.addfs(name, fs, priority=priority) startup_log.debug("%s added to templates filesystem, priority %s", fs, priority) def get_template_engine(self, engine="moya"): return self.template_engines[engine] def get_default_template_engine(self, app): engine = app.lib.templates_info.get('default_engine', 'moya') return engine def init_settings(self, cfg=None): cfg = cfg or self.cfg self.secret = cfg.get('project', 'secret', '') self.preflight = cfg.get_bool('project', 'preflight', False) self.debug = cfg.get_bool('project', 'debug') self.develop = cfg.get_bool('project', 'develop') self.log_signals = cfg.get_bool('project', 'log_signals') self.debug_echo = cfg.get_bool('project', 'debug_echo') if 'console' in cfg: self.log_logger = cfg.get('console', 'logger', None) self.log_color = cfg.get_bool('console', 'color', True) self.log_width = cfg.get_int('console', 'width', None) self.console = self.create_console() self.sites.set_defaults(cfg['site']) if 'templates' not in self.caches: self.caches['templates'] = Cache.create('templates', SettingsSectionContainer({'type': 'dict'})) if 'runtime' not in self.caches: self.caches['runtime'] = Cache.create('runtime', SettingsSectionContainer({'type': 'dict'})) require_name = ['app', 'smtp', 'db'] self.auto_reload = cfg.get_bool('autoreload', 'enabled') for section_name, section in iteritems(cfg): section = SectionWrapper(section_name, section) if ':' in section_name: what, name = section_name.split(':', 1) else: what = section_name name = None if what in require_name and not name: raise errors.StartupFailedError('Name/text required in project settings [{section}:]'.format(section=what)) if what in ('project', 'debug', 'autoreload', 'console', ''): continue if what == "settings": if name is None: self.settings.update((k, SettingContainer(v)) for k, v in iteritems(section)) else: self.app_settings[name].update((k, SettingContainer(v)) for k, v in iteritems(section)) elif what == 'application': self.app_system_settings[name].update(section) elif what == "lib": if self.has_library(name): lib = self.get_library(name) lib.settings.update((k, SettingContainer(v)) for k, v in iteritems(section)) elif what == "fs": location = section.get("location") self.add_filesystem(name, location) elif what == "data": location = section.get("location") data_fs = self.open_fs(location) self.data_fs.addfs('archive', data_fs, priority=section.get_int('priority', 0)) elif what == "cache": self.init_cache(name, section) elif what == "templates": location = section["location"] try: priority = int(section["priority"]) except (IndexError, ValueError): priority = 0 self.init_templates(name, location, priority) elif what == "db": from .db import add_engine add_engine(self, name, section) elif what == 'media': priority = section.get_int('priority', 1) location = section["location"] static_media_fs = self.open_fs(location) media_fs = MultiFS() media_fs.addfs("static", static_media_fs, priority=priority) self.add_filesystem('media', media_fs) self.media_urls = section.get_list('url') self.media_app = section.get('app', 'media') elif what == "smtp": host = section["host"] port = section.get_int('port', 25) timeout = section.get_int('timeout', None) username = section.get('username', None) password = section.get("password", None) default = section.get_bool('default', False) sender = section.get('sender', None) server = MailServer(host, name=name, port=port, default=default, timeout=timeout, username=username, password=password, sender=sender) self.mail_servers[name] = server if self.default_mail_server is None or default: self.default_mail_server = name if default: startup_log.debug('%r (default) created', server) else: startup_log.debug('%r created', server) elif what == "site": if name: self.sites.add_from_section(name, section) else: startup_log.warn("unknown settings section: [%s]", section_name) self.init_template_engine('moya', {}) def init_media(self): if 'media' not in self.filesystems: return if not self.media_urls: if not self.media_app: raise errors.StartupFailedError("no 'url' or 'app' specified in [media] section") if self.media_app not in self.apps: startup_log.warning('app set in [media]/app has not been installed ({})'.format(self.media_app)) return try: self.media_urls = [self.apps[self.media_app].mounts[0][1]] except: raise errors.StartupFailedError('unable to detect media url! (specify in [media]/url)') for i, _url in enumerate(self.media_urls): startup_log.debug('media url #%s is %s', i, _url) media_fs = self.filesystems['media'] media_mount_fs = MountFS() for app in itervalues(self.apps): for media_name, media_sub_fs in iteritems(app.lib.media): name = "%s_%s" % (app.name, media_name) media_path = "%s-%s" % (app.name, media_name) app.media[media_name] = media_path if name in self.filesystems: mount_media = self.filesystems[name] else: mount_media = media_sub_fs if name not in self.filesystems: self.filesystems[name] = mount_media media_mount_fs.mountdir(media_path, mount_media) media_fs.addfs("media", media_mount_fs) def init_data(self): data_fs = self.data_fs for lib in itervalues(self.libs): if lib.data_fs is not None: data_fs.addfs(lib.long_name, lib.data_fs, priority=lib.data_info.get('priorty', 0)) def get_media_url(self, context, app, media, path='', url_index=None): """Get a URL to media in a given app""" if url_index is None: url_index = context.inc('._media_url_index') url_no = url_index % len(self.media_urls) if app is None: return url_join(self.media_urls[url_no] or '', path) return url_join(self.media_urls[url_no] or '', app.get_media_directory(media), path) def get_element(self, element_ref, app=None, lib=None): """Gets an element from a reference""" app_id, lib_id, name = self.parse_element_ref(element_ref) if lib_id: lib = self.get_library(lib_id) element_app = self.get_app_from_lib_default(lib) element = lib.get_named_element(name) elif app_id: try: element_app = self.apps[app_id] except KeyError: raise errors.ElementNotFoundError(element_ref, app=app, lib=lib) element = element_app.lib.get_named_element(name) else: if app is not None: element_app = app element = app.lib.get_named_element(name) elif lib is not None: element_app = app element = lib.get_named_element(name) else: raise errors.ElementNotFoundError(element_ref, app=app, lib=lib) if element is None: raise errors.ElementNotFoundError(element_ref, app=app, lib=lib) return FoundElement(element_app, element) def get_app_element(self, element_ref): element_app = self.find_app(element_ref.split('#', 1)[0]) app, element = self.get_element(element_ref, app=element_app) return app or element_app, element def resolve_template_path(self, path, app_name, base_path="/"): """Get a template path in the appropriate directory for an app""" if path.startswith('/'): return path if isinstance(app_name, Application): app = app_name else: app = self.find_app(text_type(app_name)) template_path = abspath(pathjoin(base_path, app.templates_directory, path)) return template_path def get_template_lib(self, path, _cache=LRUCache()): if path in _cache: return _cache[path] lib = None for app in itervalues(self.apps): if path.startswith(app.templates_directory): lib = _cache[path] = app.lib.long_name break return lib def get_elements_by_type(self, ns, type): """Get all elements of a given namespace, type""" _elements = [] extend = _elements.extend element_type = (ns, type) for lib in itervalues(self.libs): extend(lib.get_elements_by_type(element_type)) return _elements def add_enum(self, enum): """Add an enumeration""" self.enum[enum.name] = enum def get_enum(self, enum_libid): """Get an enumeration""" return self.enum[enum_libid] def fire(self, context, signal_name, app=None, sender=None, data=None): """Fire a signal""" if data is None: data = {} signal_obj = { "name": signal_name, "app": app, "sender": sender, "data": data } if self.log_signals: _params = context.to_expr(data) if sender: signal_log.debug('firing "%s" from "%s" %s', signal_name, sender, _params) else: signal_log.debug('firing "%s" %s', signal_name, _params) for element_ref in self.signals.filter_handlers(signal_name, sender): _, libname, _ = self.parse_element_ref(element_ref) for app_name in self.apps_by_lib[libname]: app = self.apps[app_name] try: _callable = self.get_callable(element_ref, app=app) _callable(context, signal=signal_obj) except errors.LogicError as e: # We can't risk any unhandled exceptions here try: log.error("%s unhandled in signal '%s'", e, signal_name) except: pass try: if context['.debug']: context['.console'].obj(context, e) except: pass def get_callable(self, element_ref, app=None, breakpoint=False): ref_app, element = self.get_element(element_ref, app=app) if element is None: raise errors.ElementNotFoundError(element_ref) return CallableElement(self, element, ref_app or app, breakpoint=breakpoint) def get_callable_from_element(self, element, app=None, breakpoint=False): return CallableElement(self, element, app, breakpoint=breakpoint) def call(self, element_ref, context, app, *args, **kwargs): _callable = self.get_callable(element_ref, app=app) return _callable(context, *args, **kwargs) def debug_call(self, element_ref, context, app, *args, **kwargs): _callable = self.get_callable(element_ref, app=app, breakpoint=True) return _callable(context, *args, **kwargs) __call__ = call
def _get_fs(self): mfs = MultiFS() profiles = get_blob_fs("profiles") mfs.add_fs("profile_fs", profiles, priority=2) mfs.add_fs("local_fs", self.local_fs, write=True, priority=3) return mfs
def build( fs, settings_path="settings.ini", rebuild=False, archive=None, strict=False, master_settings=None, test_build=False, develop=False, ): """Build a project""" if isinstance(fs, string_types): if "://" in fs: fs = open_fs(fs) else: fs = OSFS(fs) if isinstance(settings_path, string_types): settings_path = [settings_path] try: syspath = fs.getsyspath("/") except NoSysPath: syspath = None cwd = os.getcwd() if syspath is not None: os.chdir(syspath) try: log.debug("reading settings from {}".format( textual_list(settings_path))) cfg = SettingsContainer.read(fs, settings_path, master=master_settings) if "customize" in cfg: customize_location = cfg.get("customize", "location") if customize_location: settings_path = cfg.get("customize", "settings", "settings.ini") startup_log.info("customizing '%s'", customize_location) customize_fs = open_fs(cfg.get("customize", "location")) cfg = SettingsContainer.read(customize_fs, settings_path, master=cfg) overlay_fs = MultiFS() overlay_fs.add_fs("project", fs) overlay_fs.add_fs("custom", customize_fs, write=True) fs = overlay_fs try: syspath = fs.getsyspath("/") except NoSysPath: pass else: os.chdir(syspath) if archive is None: archive = Archive(fs, strict=strict, test_build=test_build, develop=develop) context = Context() archive.cfg = cfg root = context.root root["libs"] = archive.libs root["apps"] = archive.apps root["fs"] = FSWrapper(fs) root["settings"] = SettingsContainer.from_dict(archive.cfg["settings"]) startup_path = archive.cfg.get("project", "startup") docs_location = archive.cfg.get("project", "location") archive.init_settings() root["console"] = archive.console root["debug"] = archive.debug root["_rebuild"] = rebuild parser = Parser(archive, fs.opendir(docs_location), startup_path) doc = parser.parse() if doc is None: raise errors.StartupFailedError( 'unable to parse "{}"'.format(startup_path)) archive.build(doc, fs=fs) return fs, archive, context, doc finally: os.chdir(cwd) gc.collect()
def get_multi_fs(directories): filesystem = MultiFS() for directory in directories: filesystem.add_fs(directory, fs.open_fs(directory)) return filesystem
def test_no_writable(self): fs = MultiFS() with self.assertRaises(errors.ResourceReadOnly): fs.setbytes('foo', b'bar')
def test_priority(self): """Test priority order is working""" m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.setcontents("name", b("m1")) m2.setcontents("name", b("m2")) m3.setcontents("name", b("m3")) multi_fs = MultiFS(auto_close=False) multi_fs.addfs("m1", m1) multi_fs.addfs("m2", m2) multi_fs.addfs("m3", m3) self.assert_(multi_fs.getcontents("name") == b("m3")) m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.setcontents("name", b("m1")) m2.setcontents("name", b("m2")) m3.setcontents("name", b("m3")) multi_fs = MultiFS(auto_close=False) multi_fs.addfs("m1", m1) multi_fs.addfs("m2", m2, priority=10) multi_fs.addfs("m3", m3) self.assert_(multi_fs.getcontents("name") == b("m2")) m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.setcontents("name", b("m1")) m2.setcontents("name", b("m2")) m3.setcontents("name", b("m3")) multi_fs = MultiFS(auto_close=False) multi_fs.addfs("m1", m1) multi_fs.addfs("m2", m2, priority=10) multi_fs.addfs("m3", m3, priority=10) self.assert_(multi_fs.getcontents("name") == b("m3")) m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.setcontents("name", b("m1")) m2.setcontents("name", b("m2")) m3.setcontents("name", b("m3")) multi_fs = MultiFS(auto_close=False) multi_fs.addfs("m1", m1, priority=11) multi_fs.addfs("m2", m2, priority=10) multi_fs.addfs("m3", m3, priority=10) self.assert_(multi_fs.getcontents("name") == b("m1"))
def init_settings(self, cfg=None): cfg = cfg or self.cfg self.secret = cfg.get('project', 'secret', '') self.preflight = cfg.get_bool('project', 'preflight', False) self.debug = cfg.get_bool('project', 'debug') self.develop = cfg.get_bool('project', 'develop') self.log_signals = cfg.get_bool('project', 'log_signals') self.debug_echo = cfg.get_bool('project', 'debug_echo') if 'console' in cfg: self.log_logger = cfg.get('console', 'logger', None) self.log_color = cfg.get_bool('console', 'color', True) self.log_width = cfg.get_int('console', 'width', None) self.console = self.create_console() self.sites.set_defaults(cfg['site']) if 'templates' not in self.caches: self.caches['templates'] = Cache.create('templates', SettingsSectionContainer({'type': 'dict'})) if 'runtime' not in self.caches: self.caches['runtime'] = Cache.create('runtime', SettingsSectionContainer({'type': 'dict'})) require_name = ['app', 'smtp', 'db'] self.auto_reload = cfg.get_bool('autoreload', 'enabled') for section_name, section in iteritems(cfg): section = SectionWrapper(section_name, section) if ':' in section_name: what, name = section_name.split(':', 1) else: what = section_name name = None if what in require_name and not name: raise errors.StartupFailedError('Name/text required in project settings [{section}:]'.format(section=what)) if what in ('project', 'debug', 'autoreload', 'console', ''): continue if what == "settings": if name is None: self.settings.update((k, SettingContainer(v)) for k, v in iteritems(section)) else: self.app_settings[name].update((k, SettingContainer(v)) for k, v in iteritems(section)) elif what == 'application': self.app_system_settings[name].update(section) elif what == "lib": if self.has_library(name): lib = self.get_library(name) lib.settings.update((k, SettingContainer(v)) for k, v in iteritems(section)) elif what == "fs": location = section.get("location") self.add_filesystem(name, location) elif what == "data": location = section.get("location") data_fs = self.open_fs(location) self.data_fs.addfs('archive', data_fs, priority=section.get_int('priority', 0)) elif what == "cache": self.init_cache(name, section) elif what == "templates": location = section["location"] try: priority = int(section["priority"]) except (IndexError, ValueError): priority = 0 self.init_templates(name, location, priority) elif what == "db": from .db import add_engine add_engine(self, name, section) elif what == 'media': priority = section.get_int('priority', 1) location = section["location"] static_media_fs = self.open_fs(location) media_fs = MultiFS() media_fs.addfs("static", static_media_fs, priority=priority) self.add_filesystem('media', media_fs) self.media_urls = section.get_list('url') self.media_app = section.get('app', 'media') elif what == "smtp": host = section["host"] port = section.get_int('port', 25) timeout = section.get_int('timeout', None) username = section.get('username', None) password = section.get("password", None) default = section.get_bool('default', False) sender = section.get('sender', None) server = MailServer(host, name=name, port=port, default=default, timeout=timeout, username=username, password=password, sender=sender) self.mail_servers[name] = server if self.default_mail_server is None or default: self.default_mail_server = name if default: startup_log.debug('%r (default) created', server) else: startup_log.debug('%r created', server) elif what == "site": if name: self.sites.add_from_section(name, section) else: startup_log.warn("unknown settings section: [%s]", section_name) self.init_template_engine('moya', {})
def test_priority(self): """Test priority order is working""" m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.writebytes("name", b"m1") m2.writebytes("name", b"m2") m3.writebytes("name", b"m3") multi_fs = MultiFS(auto_close=False) multi_fs.add_fs("m1", m1) multi_fs.add_fs("m2", m2) multi_fs.add_fs("m3", m3) self.assertEqual(multi_fs.readbytes("name"), b"m3") m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.writebytes("name", b"m1") m2.writebytes("name", b"m2") m3.writebytes("name", b"m3") multi_fs = MultiFS(auto_close=False) multi_fs.add_fs("m1", m1) multi_fs.add_fs("m2", m2, priority=10) multi_fs.add_fs("m3", m3) self.assertEqual(multi_fs.readbytes("name"), b"m2") m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.writebytes("name", b"m1") m2.writebytes("name", b"m2") m3.writebytes("name", b"m3") multi_fs = MultiFS(auto_close=False) multi_fs.add_fs("m1", m1) multi_fs.add_fs("m2", m2, priority=10) multi_fs.add_fs("m3", m3, priority=10) self.assertEqual(multi_fs.readbytes("name"), b"m3") m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.writebytes("name", b"m1") m2.writebytes("name", b"m2") m3.writebytes("name", b"m3") multi_fs = MultiFS(auto_close=False) multi_fs.add_fs("m1", m1, priority=11) multi_fs.add_fs("m2", m2, priority=10) multi_fs.add_fs("m3", m3, priority=10) self.assertEqual(multi_fs.readbytes("name"), b"m1")
def open_file( authority, cache, update, version_check, hasher, read_path, write_path=None, cache_on_write=False, mode='r', *args, **kwargs): ''' Context manager for reading/writing an archive and uploading on changes Parameters ---------- authority : object :py:mod:`pyFilesystem` filesystem object to use as the authoritative, up-to-date source for the archive cache : object :py:mod:`pyFilesystem` filesystem object to use as the cache. Default ``None``. use_cache : bool update, service_path, version_check, \*\*kwargs ''' if write_path is None: write_path = read_path with _choose_read_fs( authority, cache, read_path, version_check, hasher) as read_fs: write_mode = ('w' in mode) or ('a' in mode) or ('+' in mode) if write_mode: readwrite_mode = ( ('a' in mode) or ( ('r' in mode) and ( '+' in mode))) with _prepare_write_fs( read_fs, cache, read_path, readwrite_mode) as write_fs: wrapper = MultiFS() wrapper.addfs('reader', read_fs) wrapper.setwritefs(write_fs) with wrapper.open(read_path, mode, *args, **kwargs) as f: yield f info = write_fs.getinfokeys(read_path, 'size') if 'size' in info: if info['size'] == 0: return with write_fs.open(read_path, 'rb') as f: checksum = hasher(f) if not version_check(checksum): if ( cache_on_write or ( cache and ( fs.path.abspath(read_path) == fs.path.abspath(write_path)) and cache.fs.isfile(read_path) ) ): _makedirs(cache.fs, fs.path.dirname(write_path)) fs.utils.copyfile( write_fs, read_path, cache.fs, write_path) _makedirs(authority.fs, fs.path.dirname(write_path)) fs.utils.copyfile( cache.fs, write_path, authority.fs, write_path) else: _makedirs(authority.fs, fs.path.dirname(write_path)) fs.utils.copyfile( write_fs, read_path, authority.fs, write_path) update(**checksum) else: with read_fs.open(read_path, mode, *args, **kwargs) as f: yield f
def setUp(self): fs = MultiFS() mem_fs = MemoryFS() fs.add_fs("mem", mem_fs, write=True) self.fs = fs self.mem_fs = mem_fs
def test_priority(self): """Test priority order is working""" m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.setcontents("name", "m1") m2.setcontents("name", "m2") m3.setcontents("name", "m3") multi_fs = MultiFS(auto_close=False) multi_fs.addfs("m1", m1) multi_fs.addfs("m2", m2) multi_fs.addfs("m3", m3) self.assert_(multi_fs.getcontents("name") == "m3") m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.setcontents("name", "m1") m2.setcontents("name", "m2") m3.setcontents("name", "m3") multi_fs = MultiFS(auto_close=False) multi_fs.addfs("m1", m1) multi_fs.addfs("m2", m2, priority=10) multi_fs.addfs("m3", m3) self.assert_(multi_fs.getcontents("name") == "m2") m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.setcontents("name", "m1") m2.setcontents("name", "m2") m3.setcontents("name", "m3") multi_fs = MultiFS(auto_close=False) multi_fs.addfs("m1", m1) multi_fs.addfs("m2", m2, priority=10) multi_fs.addfs("m3", m3, priority=10) self.assert_(multi_fs.getcontents("name") == "m3") m1 = MemoryFS() m2 = MemoryFS() m3 = MemoryFS() m1.setcontents("name", "m1") m2.setcontents("name", "m2") m3.setcontents("name", "m3") multi_fs = MultiFS(auto_close=False) multi_fs.addfs("m1", m1, priority=11) multi_fs.addfs("m2", m2, priority=10) multi_fs.addfs("m3", m3, priority=10) self.assert_(multi_fs.getcontents("name") == "m1")
def test_no_writable(self): fs = MultiFS() with self.assertRaises(errors.ResourceReadOnly): fs.writebytes("foo", b"bar")
def __init__(self, **kwargs): self.fs = MultiFS() self.config = kwargs['config'] self.basepath = kwargs['basepath']
def init_settings(self, cfg=None): cfg = cfg or self.cfg self.secret = cfg.get("project", "secret", "") self.preflight = cfg.get_bool("project", "preflight", False) self.debug = cfg.get_bool("project", "debug") self.strict = self.strict or cfg.get_bool("project", "strict") self.develop = self.develop or cfg.get_bool("project", "develop") self.log_signals = cfg.get_bool("project", "log_signals") self.debug_echo = cfg.get_bool("project", "debug_echo") self.debug_memory = cfg.get_bool("project", "debug_memory") self.lib_paths = cfg.get_list("project", "paths", "./local\n./external") self.lib_paths = self.lib_paths[:] + [MOYA_LIBS_PATH] if "console" in cfg: self.log_logger = cfg.get("console", "logger", None) self.log_color = cfg.get_bool("console", "color", True) self.log_width = cfg.get_int("console", "width", None) self.console = self.create_console() self.sites.set_defaults(cfg["site"]) if "templates" not in self.caches: self.caches["templates"] = Cache.create( "templates", SettingsSectionContainer({"type": "dict"}) ) if "runtime" not in self.caches: self.caches["runtime"] = Cache.create( "runtime", SettingsSectionContainer({"type": "dict"}) ) require_name = ["app", "smtp", "db"] self.auto_reload = cfg.get_bool("autoreload", "enabled") if self.strict: startup_log.debug("strict mode is enabled") if self.develop: startup_log.debug("develop mode is enabled") for section_name, section in iteritems(cfg): section = SectionWrapper(section_name, section) if ":" in section_name: what, name = section_name.split(":", 1) else: what = section_name name = None if what in require_name and not name: raise errors.StartupFailedError( "name required in section, [{section}:?]".format(section=what) ) if what in ("project", "debug", "autoreload", "console", "customize", ""): continue if what == "settings": if name is None: self.settings.update( (k, SettingContainer(v)) for k, v in iteritems(section) ) else: self.app_settings[name].update( (k, SettingContainer(v)) for k, v in iteritems(section) ) elif what == "application": self.app_system_settings[name].update(section) elif what == "lib": if self.has_library(name): lib = self.get_library(name) lib.settings.update( (k, SettingContainer(v)) for k, v in iteritems(section) ) elif what == "fs": location = section.get("location") if not location: raise errors.StartupFailedError( "a value for 'location' is required in [{}]".format( section_name ) ) create = section.get_bool("create", False) self.add_filesystem(name, location, create=create) elif what == "data": location = section.get("location") data_fs = self.open_fs(location) self.data_fs.add_fs( "archive", data_fs, priority=section.get_int("priority", 0) ) elif what == "cache": self.init_cache(name, section) elif what == "templates": location = section["location"] try: priority = int(section["priority"]) except (IndexError, ValueError): priority = 0 self.init_templates(name, location, priority) elif what == "db": from .db import add_engine add_engine(self, name, section) elif what == "media": priority = section.get_int("priority", 1) location = section["location"] static_media_fs = self.open_fs(location) media_fs = MultiFS() media_fs.add_fs("static", static_media_fs, priority=priority) self.add_filesystem("media", media_fs) self.media_urls = section.get_list("url") self.media_app = section.get("app", "media") elif what == "smtp": host = section["host"] port = section.get_int("port", 25) timeout = section.get_int("timeout", None) username = section.get("username", None) password = section.get("password", None) default = section.get_bool("default", False) sender = section.get("sender", None) server = MailServer( host, name=name, port=port, default=default, timeout=timeout, username=username, password=password, sender=sender, ) self.mail_servers[name] = server if self.default_mail_server is None or default: self.default_mail_server = name if default: startup_log.debug("%r (default) created", server) else: startup_log.debug("%r created", server) elif what == "site": if name: self.sites.add_from_section(name, section) elif what == "themes": location = section["location"] theme_fs = self.open_fs(location) self.add_filesystem("themes", theme_fs) # startup_log.debug("added theme filesystem '%s'", location) else: startup_log.warn("unknown settings section, [%s]", section_name) self.init_template_engine("moya", {})
def _get_fs(self, fs_url): mfs = MultiFS() mfs.add_fs("remotefs", fs.open_fs(fs_url), write=True, priority=3) return mfs
def make_fs(self): fs = MultiFS() mem_fs = MemoryFS() fs.add_fs("mem", mem_fs, write=True) return fs