def test_no_auto_close(self):
"""Test MultiFS auto close can be disables"""
multi_fs = MultiFS(auto_close=False)
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.addfs('m1', m1)
multi_fs.addfs('m2', m2)
self.assert_(not m1.closed)
self.assert_(not m2.closed)
multi_fs.close()
self.assert_(not m1.closed)
self.assert_(not m2.closed)
def test_auto_close(self):
"""Test MultiFS auto close is working"""
multi_fs = MultiFS()
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.addfs('m1', m1)
multi_fs.addfs('m2', m2)
self.assert_(not m1.closed)
self.assert_(not m2.closed)
multi_fs.close()
self.assert_(m1.closed)
self.assert_(m2.closed)
def test_no_auto_close(self):
"""Test MultiFS auto close can be disables"""
multi_fs = MultiFS(auto_close=False)
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.addfs('m1', m1)
multi_fs.addfs('m2', m2)
self.assert_(not m1.closed)
self.assert_(not m2.closed)
multi_fs.close()
self.assert_(not m1.closed)
self.assert_(not m2.closed)
def test_auto_close(self):
"""Test MultiFS auto close is working"""
multi_fs = MultiFS()
m1 = MemoryFS()
m2 = MemoryFS()
multi_fs.addfs('m1', m1)
multi_fs.addfs('m2', m2)
self.assert_(not m1.closed)
self.assert_(not m2.closed)
multi_fs.close()
self.assert_(m1.closed)
self.assert_(m2.closed)
class _AnsiblePlaybookSources(object):
def __init__(self, **kwargs):
self.fs = MultiFS()
self.config = kwargs['config']
self.basepath = kwargs['basepath']
def add_folder(self, folder_path, dest=""):
real_path = search_path(folder_path, self.basepath, getcwd())
if real_path is None:
raise RuntimeError("Couldn't find ansible playbook sources at %s" % folder_path)
self.fs.addfs(dest, OSFS(real_path))
# TODO: In the future...
# def add_git(self, **kwargs):
# def add_file_url(self, url):
# Plain method to embed the playbook contents into the cloudformation template
def _fs_to_cfninit_plain(self):
from base64 import b64encode
# Create a metadata object that, when executed by cfn-init, will result in the files
# being created in the instance
from os.path import join
files_obj = {}
target = self.config['_inst_playbook_path']
for (fs_dir, fs_dirfiles) in self.fs.walk():
# TODO: more deterministic order of walking through the files
if fs_dir[0] == '/': fs_dir = fs_dir[1:]
t = join(target, fs_dir)
for f in fs_dirfiles:
src = join(fs_dir, f)
dest = join(t, f)
encoding= None
try:
contents= self.fs.getcontents(src, mode='rb')
contents.decode('utf-8')
except UnicodeDecodeError:
encoding= "base64"
contents= b64encode(contents)
files_obj[dest] = dict(
content= contents,
owner= self.config['inst_user'],
group= self.config['inst_group'],
mode= "000640"
)
files_obj[dest].update( {"encoding": encoding} if encoding else {} )
return files_obj
def deploy(self, iscm):
# Create cfn-init stanzas for creating the added files
iscm.iscm_cfninit_add_config({ "files": self._fs_to_cfninit_plain() }, "_ansible_playbooks_install")
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir):
from fs.multifs import MultiFS
from configparser import ConfigParser
cfg = ConfigParser()
if '#' in fs_path:
path, section = fs_path.split('#', 1)
else:
path = fs_path
section = 'fs'
cfg.readfp(registry.open(path))
multi_fs = MultiFS()
for name, fs_url in cfg.items(section):
multi_fs.addfs(name, registry.opendir(fs_url, create_dir=create_dir))
return multi_fs, ''
def get_fs(cls, registry, fs_name, fs_name_params, fs_path, writeable, create_dir):
from fs.multifs import MultiFS
from ConfigParser import ConfigParser
cfg = ConfigParser()
if '#' in fs_path:
path, section = fs_path.split('#', 1)
else:
path = fs_path
section = 'fs'
cfg.readfp(registry.open(path))
multi_fs = MultiFS()
for name, fs_url in cfg.items(section):
multi_fs.addfs(name, registry.opendir(fs_url, create_dir=create_dir))
return multi_fs, ''
def _find_extensions(paths):
'''Iterate the paths, in order, finding extensions and adding them to
the return dict.'''
extension_kinds = ['check', 'configure', 'write']
efs = MultiFS()
map(lambda x: efs.addfs(x, OSFS(x)), paths)
def get_extensions(kind):
return {os.path.splitext(x)[0]: efs.getsyspath(x)
for x in efs.walkfiles('.', '*.%s' % kind)}
return {e: get_extensions(e) for e in extension_kinds}
def _find_extensions(paths):
'''Iterate the paths, in order, finding extensions and adding them to
the return dict.'''
extension_kinds = ['check', 'configure', 'write']
efs = MultiFS()
map(lambda x: efs.addfs(x, OSFS(x)), paths)
def get_extensions(kind):
return {os.path.splitext(x)[0]: efs.getsyspath(x)
for x in efs.walkfiles('.', '*.%s' % kind)}
return {e: get_extensions(e) for e in extension_kinds}
def test_priority(self):
"""Test priority order is working"""
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2)
multi_fs.addfs("m3", m3)
self.assert_(multi_fs.getcontents("name") == "m3")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3)
self.assert_(multi_fs.getcontents("name") == "m2")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3, priority=10)
self.assert_(multi_fs.getcontents("name") == "m3")
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", "m1")
m2.setcontents("name", "m2")
m3.setcontents("name", "m3")
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1, priority=11)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3, priority=10)
self.assert_(multi_fs.getcontents("name") == "m1")
def test_priority(self):
"""Test priority order is working"""
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", b("m1"))
m2.setcontents("name", b("m2"))
m3.setcontents("name", b("m3"))
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2)
multi_fs.addfs("m3", m3)
self.assert_(multi_fs.getcontents("name") == b("m3"))
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", b("m1"))
m2.setcontents("name", b("m2"))
m3.setcontents("name", b("m3"))
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3)
self.assert_(multi_fs.getcontents("name") == b("m2"))
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", b("m1"))
m2.setcontents("name", b("m2"))
m3.setcontents("name", b("m3"))
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3, priority=10)
self.assert_(multi_fs.getcontents("name") == b("m3"))
m1 = MemoryFS()
m2 = MemoryFS()
m3 = MemoryFS()
m1.setcontents("name", b("m1"))
m2.setcontents("name", b("m2"))
m3.setcontents("name", b("m3"))
multi_fs = MultiFS(auto_close=False)
multi_fs.addfs("m1", m1, priority=11)
multi_fs.addfs("m2", m2, priority=10)
multi_fs.addfs("m3", m3, priority=10)
self.assert_(multi_fs.getcontents("name") == b("m1"))
def init_settings(self, cfg=None):
cfg = cfg or self.cfg
self.secret = cfg.get('project', 'secret', '')
self.preflight = cfg.get_bool('project', 'preflight', False)
self.debug = cfg.get_bool('project', 'debug')
self.develop = cfg.get_bool('project', 'develop')
self.log_signals = cfg.get_bool('project', 'log_signals')
self.debug_echo = cfg.get_bool('project', 'debug_echo')
if 'console' in cfg:
self.log_logger = cfg.get('console', 'logger', None)
self.log_color = cfg.get_bool('console', 'color', True)
self.log_width = cfg.get_int('console', 'width', None)
self.console = self.create_console()
self.sites.set_defaults(cfg['site'])
if 'templates' not in self.caches:
self.caches['templates'] = Cache.create('templates', SettingsSectionContainer({'type': 'dict'}))
if 'runtime' not in self.caches:
self.caches['runtime'] = Cache.create('runtime', SettingsSectionContainer({'type': 'dict'}))
require_name = ['app', 'smtp', 'db']
self.auto_reload = cfg.get_bool('autoreload', 'enabled')
for section_name, section in iteritems(cfg):
section = SectionWrapper(section_name, section)
if ':' in section_name:
what, name = section_name.split(':', 1)
else:
what = section_name
name = None
if what in require_name and not name:
raise errors.StartupFailedError('Name/text required in project settings [{section}:]'.format(section=what))
if what in ('project', 'debug', 'autoreload', 'console', ''):
continue
if what == "settings":
if name is None:
self.settings.update((k, SettingContainer(v))
for k, v in iteritems(section))
else:
self.app_settings[name].update((k, SettingContainer(v))
for k, v in iteritems(section))
elif what == 'application':
self.app_system_settings[name].update(section)
elif what == "lib":
if self.has_library(name):
lib = self.get_library(name)
lib.settings.update((k, SettingContainer(v))
for k, v in iteritems(section))
elif what == "fs":
location = section.get("location")
self.add_filesystem(name, location)
elif what == "data":
location = section.get("location")
data_fs = self.open_fs(location)
self.data_fs.addfs('archive',
data_fs,
priority=section.get_int('priority', 0))
elif what == "cache":
self.init_cache(name, section)
elif what == "templates":
location = section["location"]
try:
priority = int(section["priority"])
except (IndexError, ValueError):
priority = 0
self.init_templates(name, location, priority)
elif what == "db":
from .db import add_engine
add_engine(self, name, section)
elif what == 'media':
priority = section.get_int('priority', 1)
location = section["location"]
static_media_fs = self.open_fs(location)
media_fs = MultiFS()
media_fs.addfs("static", static_media_fs, priority=priority)
self.add_filesystem('media', media_fs)
self.media_urls = section.get_list('url')
self.media_app = section.get('app', 'media')
elif what == "smtp":
host = section["host"]
port = section.get_int('port', 25)
timeout = section.get_int('timeout', None)
username = section.get('username', None)
password = section.get("password", None)
default = section.get_bool('default', False)
sender = section.get('sender', None)
server = MailServer(host,
name=name,
port=port,
default=default,
timeout=timeout,
username=username,
password=password,
sender=sender)
self.mail_servers[name] = server
if self.default_mail_server is None or default:
self.default_mail_server = name
if default:
startup_log.debug('%r (default) created', server)
else:
startup_log.debug('%r created', server)
elif what == "site":
if name:
self.sites.add_from_section(name, section)
else:
startup_log.warn("unknown settings section: [%s]", section_name)
self.init_template_engine('moya', {})
class Archive(object):
_re_element_ref_match = re.compile(r'^(.+\..+)#(.*)$|^(.+)#(.*)$|^#(.+)$', re.UNICODE).match
def __init__(self, project_fs=None, breakpoint=False):
self.project_fs = project_fs
self.registry = ElementRegistry()
self.libs = {}
self.apps = OrderedDict()
self.apps_by_lib = defaultdict(list)
self.app_settings = defaultdict(SettingsContainer)
self.app_system_settings = defaultdict(SettingsContainer)
self.cfg = None
self.settings = SettingsContainer()
self.templates_fs = MultiFS()
self.data_fs = MultiFS()
self.filesystems = FSContainer({'templates': self.templates_fs,
'data': self.data_fs})
self.filters = FilterContainer(self)
self.template_engines = {}
self.database_engines = {}
self.caches = {}
self.mail_servers = {}
self.default_mail_server = None
self.default_db_engine = None
self.debug = False
self.auto_reload = False
self.known_namespaces = set()
self.sites = Sites()
self.breakpoint = breakpoint
self.suppress_breakpoints = False
self.data_tags = defaultdict(list)
self.data_tags_by_lib = defaultdict(lambda: defaultdict(list)) # awesome
self.preflight = False
self.log_signals = False
self.debug_echo = False
self.log_logger = None
self.log_color = True
self.log_width = None
self.media_urls = None
self.media_app = None
self.failed_documents = []
self.enum = {}
self.signals = Signals()
self._moyarc = None
self.console = self.create_console()
def __repr__(self):
return "<archive>"
@property
def media_url(self):
return self.media_urls[0] if self.media_urls else None
@property
def moyarc(self):
if self._moyarc is None:
try:
with io.open(os.path.expanduser("~/.moyarc"), 'rt') as f:
self._moyarc = settings.SettingsContainer.read_from_file(f)
except IOError:
self._moyarc = settings.SettingsContainer()
return self._moyarc
def open_fs(self, fs_url):
if isinstance(fs_url, text_type):
if '://' in fs_url:
fs = fsopendir(fs_url)
else:
fs = self.project_fs.opendir(fs_url)
else:
fs = fs_url
return fs
def get_console_file(self):
if self.log_logger:
console_file = logtools.LoggerFile(self.log_logger)
else:
console_file = None
return console_file
def create_console(self):
console = Console(out=self.get_console_file(),
nocolors=not (self.log_color and self.moyarc.get_bool('console', 'color', True)),
width=self.log_width or None)
return console
def build_libs(self, ignore_errors=False):
libs = [lib for lib in itervalues(self.libs) if not lib.built]
if not libs:
return
start = time()
self.build([doc for lib in libs
for doc in lib.documents],
log_time=False)
for lib in libs:
lib.finalize(ignore_errors=ignore_errors)
startup_log.debug("%s built libraries %0.1fms",
self,
(time() - start) * 1000.0)
def build(self, documents, context=None, log_time=True, fs=None):
"""Build all documents in the library"""
# This handles tags defined out of order
if fs is not None:
self.project_fs = fs
start = time()
if isinstance(documents, Document):
documents = [documents]
else:
documents = list(documents)
build_queue = deque()
build_queue_appendleft = build_queue.appendleft
for doc in documents:
if not doc.document_finalized and doc.structure:
build_queue.append(_BuildFrame([doc.structure.root_node]))
unbuildable = set()
unbuildable_clear = unbuildable.clear
unbuilt = []
if context is None:
context = Context()
context_root = context.root
get_doc_id = attrgetter('doc_id')
while build_queue:
nodes = build_queue[0]
if nodes:
node = nodes.pop()
context_root['_lib_long_name'] = node.lib_long_name
element = node.build(self, context)
if element:
unbuildable_clear()
build_queue_appendleft(_BuildFrame(sorted(node.children, key=get_doc_id, reverse=True), element=element))
else:
if element is not None:
if node in unbuildable:
# A previously deferred node, but we still can't built it
build_queue.popleft() # Can't process siblings either
unbuilt.append(node)
else:
# Defer this node till later
nodes.append(node)
build_queue.rotate(-1)
unbuildable.add(node)
else:
if nodes.element is not None:
unbuildable_clear()
try:
nodes.element.finalize(context)
except Exception as e:
raise
failed_doc = FailedDocument(path=node.structure.document.location,
code=node.structure.xml,
line=node.source_line,
col=None,
msg=text_type(e))
self.failed_documents.append(failed_doc)
#return False
build_queue.popleft()
if unbuilt:
for node in unbuilt:
nearest = nearest_word(node.tag_name,
self.registry.get_elements_in_xmlns(node.xmlns))
msg = "unknown tag {} in {}".format(node.tag_display_name, node.structure)
diagnosis = None
if nearest:
diagnosis = "did you mean <{}>?".format(nearest)
else:
find_xmlns = self.registry.find_xmlns(node.tag_name)
if find_xmlns:
diagnosis = "did you mean <{}> in XMLNS '{}'?".format(node.tag_name, find_xmlns)
failed_doc = FailedDocument(path=node.structure.document.location,
code=node.structure.xml,
line=node.source_line,
col=None,
msg=msg,
diagnosis=diagnosis)
self.failed_documents.append(failed_doc)
return False
for doc in documents:
doc.document_finalize(context)
if log_time:
doc_text = ', '.join(text_type(doc) for doc in documents)
startup_log.debug("%s built %0.1fms",
doc_text,
(time() - start) * 1000.0)
return True
def populate_context(self, context):
from .context.expressiontime import ExpressionDateTime
root = context.root
root['libs'] = self.libs
root['apps'] = self.apps
root['filters'] = self.filters
root['debug'] = self.debug
root['develop'] = self.develop
root['fs'] = self.get_context_filesystems()
root['now'] = ExpressionDateTime.utcnow()
from . import __version__
root['moya'] = {'version': __version__}
root['enum'] = self.enum
root['media_url'] = self.media_url
root['secret'] = self.secret
context.set_dynamic('.app', lambda context: getattr(context.get('.call', None), 'app'))
@classmethod
def get_callable_from_document(cls, path, element_ref, breakpoint=False, fs='./', default_context=False, archive=None, lib=None):
"""Shortcut that imports a single document and returns a callable"""
if archive is None:
archive = cls()
if lib is None:
lib = archive.create_library(long_name="moya.run", namespace=namespaces.run)
lib.import_document(fs, path)
archive.build_libs()
app, element = lib.documents[0].get_element(element_ref)
if element is None:
raise errors.ElementNotFoundError(element_ref, app=app)
call = CallableElement(archive, element, app, breakpoint=breakpoint)
if default_context:
def do_call(*args, **kwargs):
c = Context()
c['console'] = Console()
return call(c, *args, **kwargs)
do_call.archive = archive
return do_call
return call
@classmethod
def parse_element_ref(cls, s, cache={}):
try:
return cache[s]
except KeyError:
match = cls._re_element_ref_match(s)
if match is None:
result = None, None, s
else:
libname, lib_elementname, appname, app_elementname, docname = match.groups()
result = appname, libname, app_elementname or lib_elementname or docname
cache[s] = result
return result
def get_library(self, library_name):
"""Get a library from either its short name, or its long name"""
try:
return self.libs[library_name]
except KeyError:
raise errors.UnknownLibraryError(lib=library_name)
def has_library(self, library_name):
return library_name in self.libs
def load_library(self, import_fs, priority=None, template_priority=None, long_name=None, rebuild=False):
"""Load a new library in to the archive"""
lib = self.create_library(import_fs, long_name=long_name, rebuild=rebuild)
if priority is not None:
lib.priority = priority
if lib.templates_info:
fs_url = lib.templates_info['location']
if template_priority is None:
try:
template_priority = int(lib.templates_info.get('priority', '0'))
except ValueError:
startup_log.error("{} Invalid value for [templates]/priority, assuming 0".format(lib))
template_priority = 0
lib.template_priority = template_priority
if '://' in fs_url:
fs = fsopendir(fs_url)
else:
fs = import_fs.opendir(fs_url)
self.templates_fs.addfs(lib.long_name, fs, priority=template_priority)
return lib
def get_or_create_library(self, long_name, import_fs=None):
"""Get a library, or create it if it doesn't exists"""
if long_name is not None and long_name in self.libs:
return self.libs[long_name]
return self.create_library(import_fs)
def create_library(self, import_fs=None, long_name=None, namespace=None, rebuild=False):
"""Create a new library, and import documents"""
lib = Library(self, import_fs, long_name=long_name, namespace=namespace, rebuild=rebuild)
self.libs[lib.long_name] = lib
return lib
def finalize(self, ignore_errors=False):
self.build_libs(ignore_errors=ignore_errors)
for lib in itervalues(self.libs):
lib.on_archive_finalize()
if self.database_engines:
for app in itervalues(self.apps):
for model in app.lib.get_elements_by_type((namespaces.db, "model")):
if not model.abstract:
model.get_table_and_class(app)
def create_app(self, name, lib_name):
if name in self.apps:
raise errors.ArchiveError("Application name '{}' was previously installed with {}".format(name, self.apps[name].lib))
app = Application(self, name, lib_name)
self.apps[name] = app
app.settings.update(self.app_settings[name])
app.system_settings.update(self.app_system_settings[name])
self.apps_by_lib[lib_name].append(name)
return app
def get_app(self, app_id):
try:
return self.apps[app_id]
except IndexError:
return None
def has_app(self, app_id):
return app_id in self.apps
def get_app_from_lib(self, lib):
if isinstance(lib, Library):
lib_name = lib.long_name
else:
lib_name = text_type(lib)
if '.' not in lib_name:
return lib_name
apps = self.apps_by_lib[lib_name]
if len(apps) != 1:
if not apps:
raise errors.AppRequiredError(lib_name)
raise errors.AmbiguousAppError(lib_name, apps)
return self.apps[apps[0]]
def get_app_from_lib_default(self, lib, default=None):
if isinstance(lib, Library):
lib_name = lib.long_name
else:
lib_name = text_type(lib)
if '.' not in lib_name:
return lib_name
apps = self.apps_by_lib[lib_name]
if len(apps) != 1:
return default
return self.apps[apps[0]]
def find_app(self, name):
"""Find an app from either its name or its lib name.
If a lib name is supplied and there are more than one app, an AmbiguousAppError is raise
"""
if isinstance(name, Application):
return name
if not name:
raise errors.UnknownAppError("Value {} is not a valid app or lib name".format(to_expression(None, name)))
name = text_type(name)
try:
if '.' in name:
apps = self.apps_by_lib[name]
if not apps:
raise KeyError("No app called '{}'".format(name))
if len(apps) != 1:
raise errors.AmbiguousAppError(name, apps)
return self.apps[apps[0]]
else:
return self.apps[name]
except KeyError:
raise errors.UnknownAppError(app=name)
def get_lib(self, name):
if isinstance(name, Library):
return name.long_name
if '.' in name:
return self.libs[name].long_name
return self.find_app(name).lib.long_name
def add_data_tag(self, element_type, tag):
self.data_tags[element_type].append(tag)
self.data_tags_by_lib[tag.lib.long_name][element_type].append(tag)
def get_data(self, context, namespace, tag_name):
"""Get data from a data tag"""
tag_type = (namespace, tag_name)
return [e.get_all_data_parameters(context)
for e in self.data_tags.get(tag_type, [])
if e.check(context)]
def get_data_item(self, context, namespace, tag_name, filter_map):
"""Get data from a data tag"""
tag_type = (namespace, tag_name)
for e in self.data_tags.get(tag_type, []):
data = e.get_all_data_parameters(context)
if all(filter_map[k] == data.get(k, Ellipsis)
for k, v in filter_map.items()):
return data
def get_data_from_element(self, context, element):
return element.get_all_data_parameters(context)
def get_data_elements(self, context, namespace, tag_name):
"""Get data from a data tag"""
tag_type = (namespace, tag_name)
return [DataElement(e, context)
for e in self.data_tags.get(tag_type, [])
if e.check(context)]
def get_app_data_elements(self, context, namespace, tag_name):
data = []
tag_type = (namespace, tag_name)
for app in itervalues(self.apps):
_elements = []
append = _elements.append
for e in self.data_tags_by_lib[app.lib.long_name].get(tag_type, []):
if e.check(context):
append(DataElement(e, context))
if _elements:
data.append((app, _elements))
return data
def add_filesystem(self, name, fs):
add_fs = self.open_fs(fs)
self.filesystems[name] = add_fs
startup_log.debug("%s fs added as '%s'", add_fs, name)
return fs
def get_filesystem(self, name):
return self.filesystems[name]
def get_reader(self, name="data"):
fs = self.get_filesystem(name)
return DataReader(fs)
def get_context_filesystems(self):
return FSContainer((k, FSWrapper(fs))
for k, fs in iteritems(self.filesystems))
def get_translations(self, app_or_lib, languages):
return self.find_app(app_or_lib).lib.translations.get(languages)
def init_template_engine(self, system, settings):
if system in self.template_engines:
return
engine = TemplateEngine.create(system,
self,
self.templates_fs,
settings)
self.template_engines[system] = engine
startup_log.debug('%s template engine initialized', engine)
def init_cache(self, name, settings):
cache = Cache.create(name, settings)
self.caches[name] = cache
startup_log.debug('%s cache added', cache)
def get_cache(self, name):
if name in self.caches:
return self.caches[name]
cache = self.caches[name] = Cache.create('runtime', SettingsSectionContainer({'type': 'dict'}))
return cache
def get_mailserver(self, name=None):
name = name or self.default_mail_server or 'default'
try:
return self.mail_servers[name or 'default']
except KeyError:
raise errors.MoyaException("email.no-server", "no email server called '{0}'".format(name))
def init_templates(self, name, location, priority):
templates_fs = self.filesystems.get("templates")
fs = self.open_fs(location)
templates_fs.addfs(name, fs, priority=priority)
startup_log.debug("%s added to templates filesystem, priority %s", fs, priority)
def get_template_engine(self, engine="moya"):
return self.template_engines[engine]
def get_default_template_engine(self, app):
engine = app.lib.templates_info.get('default_engine', 'moya')
return engine
def init_settings(self, cfg=None):
cfg = cfg or self.cfg
self.secret = cfg.get('project', 'secret', '')
self.preflight = cfg.get_bool('project', 'preflight', False)
self.debug = cfg.get_bool('project', 'debug')
self.develop = cfg.get_bool('project', 'develop')
self.log_signals = cfg.get_bool('project', 'log_signals')
self.debug_echo = cfg.get_bool('project', 'debug_echo')
if 'console' in cfg:
self.log_logger = cfg.get('console', 'logger', None)
self.log_color = cfg.get_bool('console', 'color', True)
self.log_width = cfg.get_int('console', 'width', None)
self.console = self.create_console()
self.sites.set_defaults(cfg['site'])
if 'templates' not in self.caches:
self.caches['templates'] = Cache.create('templates', SettingsSectionContainer({'type': 'dict'}))
if 'runtime' not in self.caches:
self.caches['runtime'] = Cache.create('runtime', SettingsSectionContainer({'type': 'dict'}))
require_name = ['app', 'smtp', 'db']
self.auto_reload = cfg.get_bool('autoreload', 'enabled')
for section_name, section in iteritems(cfg):
section = SectionWrapper(section_name, section)
if ':' in section_name:
what, name = section_name.split(':', 1)
else:
what = section_name
name = None
if what in require_name and not name:
raise errors.StartupFailedError('Name/text required in project settings [{section}:]'.format(section=what))
if what in ('project', 'debug', 'autoreload', 'console', ''):
continue
if what == "settings":
if name is None:
self.settings.update((k, SettingContainer(v))
for k, v in iteritems(section))
else:
self.app_settings[name].update((k, SettingContainer(v))
for k, v in iteritems(section))
elif what == 'application':
self.app_system_settings[name].update(section)
elif what == "lib":
if self.has_library(name):
lib = self.get_library(name)
lib.settings.update((k, SettingContainer(v))
for k, v in iteritems(section))
elif what == "fs":
location = section.get("location")
self.add_filesystem(name, location)
elif what == "data":
location = section.get("location")
data_fs = self.open_fs(location)
self.data_fs.addfs('archive',
data_fs,
priority=section.get_int('priority', 0))
elif what == "cache":
self.init_cache(name, section)
elif what == "templates":
location = section["location"]
try:
priority = int(section["priority"])
except (IndexError, ValueError):
priority = 0
self.init_templates(name, location, priority)
elif what == "db":
from .db import add_engine
add_engine(self, name, section)
elif what == 'media':
priority = section.get_int('priority', 1)
location = section["location"]
static_media_fs = self.open_fs(location)
media_fs = MultiFS()
media_fs.addfs("static", static_media_fs, priority=priority)
self.add_filesystem('media', media_fs)
self.media_urls = section.get_list('url')
self.media_app = section.get('app', 'media')
elif what == "smtp":
host = section["host"]
port = section.get_int('port', 25)
timeout = section.get_int('timeout', None)
username = section.get('username', None)
password = section.get("password", None)
default = section.get_bool('default', False)
sender = section.get('sender', None)
server = MailServer(host,
name=name,
port=port,
default=default,
timeout=timeout,
username=username,
password=password,
sender=sender)
self.mail_servers[name] = server
if self.default_mail_server is None or default:
self.default_mail_server = name
if default:
startup_log.debug('%r (default) created', server)
else:
startup_log.debug('%r created', server)
elif what == "site":
if name:
self.sites.add_from_section(name, section)
else:
startup_log.warn("unknown settings section: [%s]", section_name)
self.init_template_engine('moya', {})
def init_media(self):
if 'media' not in self.filesystems:
return
if not self.media_urls:
if not self.media_app:
raise errors.StartupFailedError("no 'url' or 'app' specified in [media] section")
if self.media_app not in self.apps:
startup_log.warning('app set in [media]/app has not been installed ({})'.format(self.media_app))
return
try:
self.media_urls = [self.apps[self.media_app].mounts[0][1]]
except:
raise errors.StartupFailedError('unable to detect media url! (specify in [media]/url)')
for i, _url in enumerate(self.media_urls):
startup_log.debug('media url #%s is %s', i, _url)
media_fs = self.filesystems['media']
media_mount_fs = MountFS()
for app in itervalues(self.apps):
for media_name, media_sub_fs in iteritems(app.lib.media):
name = "%s_%s" % (app.name, media_name)
media_path = "%s-%s" % (app.name, media_name)
app.media[media_name] = media_path
if name in self.filesystems:
mount_media = self.filesystems[name]
else:
mount_media = media_sub_fs
if name not in self.filesystems:
self.filesystems[name] = mount_media
media_mount_fs.mountdir(media_path, mount_media)
media_fs.addfs("media", media_mount_fs)
def init_data(self):
data_fs = self.data_fs
for lib in itervalues(self.libs):
if lib.data_fs is not None:
data_fs.addfs(lib.long_name, lib.data_fs, priority=lib.data_info.get('priorty', 0))
def get_media_url(self, context, app, media, path='', url_index=None):
"""Get a URL to media in a given app"""
if url_index is None:
url_index = context.inc('._media_url_index')
url_no = url_index % len(self.media_urls)
if app is None:
return url_join(self.media_urls[url_no] or '', path)
return url_join(self.media_urls[url_no] or '',
app.get_media_directory(media),
path)
def get_element(self, element_ref, app=None, lib=None):
"""Gets an element from a reference"""
app_id, lib_id, name = self.parse_element_ref(element_ref)
if lib_id:
lib = self.get_library(lib_id)
element_app = self.get_app_from_lib_default(lib)
element = lib.get_named_element(name)
elif app_id:
try:
element_app = self.apps[app_id]
except KeyError:
raise errors.ElementNotFoundError(element_ref, app=app, lib=lib)
element = element_app.lib.get_named_element(name)
else:
if app is not None:
element_app = app
element = app.lib.get_named_element(name)
elif lib is not None:
element_app = app
element = lib.get_named_element(name)
else:
raise errors.ElementNotFoundError(element_ref, app=app, lib=lib)
if element is None:
raise errors.ElementNotFoundError(element_ref, app=app, lib=lib)
return FoundElement(element_app, element)
def get_app_element(self, element_ref):
element_app = self.find_app(element_ref.split('#', 1)[0])
app, element = self.get_element(element_ref, app=element_app)
return app or element_app, element
def resolve_template_path(self, path, app_name, base_path="/"):
"""Get a template path in the appropriate directory for an app"""
if path.startswith('/'):
return path
if isinstance(app_name, Application):
app = app_name
else:
app = self.find_app(text_type(app_name))
template_path = abspath(pathjoin(base_path, app.templates_directory, path))
return template_path
def get_template_lib(self, path, _cache=LRUCache()):
if path in _cache:
return _cache[path]
lib = None
for app in itervalues(self.apps):
if path.startswith(app.templates_directory):
lib = _cache[path] = app.lib.long_name
break
return lib
def get_elements_by_type(self, ns, type):
"""Get all elements of a given namespace, type"""
_elements = []
extend = _elements.extend
element_type = (ns, type)
for lib in itervalues(self.libs):
extend(lib.get_elements_by_type(element_type))
return _elements
def add_enum(self, enum):
"""Add an enumeration"""
self.enum[enum.name] = enum
def get_enum(self, enum_libid):
"""Get an enumeration"""
return self.enum[enum_libid]
def fire(self, context, signal_name, app=None, sender=None, data=None):
"""Fire a signal"""
if data is None:
data = {}
signal_obj = {
"name": signal_name,
"app": app,
"sender": sender,
"data": data
}
if self.log_signals:
_params = context.to_expr(data)
if sender:
signal_log.debug('firing "%s" from "%s" %s', signal_name, sender, _params)
else:
signal_log.debug('firing "%s" %s', signal_name, _params)
for element_ref in self.signals.filter_handlers(signal_name, sender):
_, libname, _ = self.parse_element_ref(element_ref)
for app_name in self.apps_by_lib[libname]:
app = self.apps[app_name]
try:
_callable = self.get_callable(element_ref, app=app)
_callable(context, signal=signal_obj)
except errors.LogicError as e:
# We can't risk any unhandled exceptions here
try:
log.error("%s unhandled in signal '%s'", e, signal_name)
except:
pass
try:
if context['.debug']:
context['.console'].obj(context, e)
except:
pass
def get_callable(self, element_ref, app=None, breakpoint=False):
ref_app, element = self.get_element(element_ref, app=app)
if element is None:
raise errors.ElementNotFoundError(element_ref)
return CallableElement(self, element, ref_app or app, breakpoint=breakpoint)
def get_callable_from_element(self, element, app=None, breakpoint=False):
return CallableElement(self, element, app, breakpoint=breakpoint)
def call(self, element_ref, context, app, *args, **kwargs):
_callable = self.get_callable(element_ref, app=app)
return _callable(context, *args, **kwargs)
def debug_call(self, element_ref, context, app, *args, **kwargs):
_callable = self.get_callable(element_ref, app=app, breakpoint=True)
return _callable(context, *args, **kwargs)
__call__ = call
def open_file(
authority,
cache,
update,
version_check,
hasher,
read_path,
write_path=None,
cache_on_write=False,
mode='r',
*args,
**kwargs):
'''
Context manager for reading/writing an archive and uploading on changes
Parameters
----------
authority : object
:py:mod:`pyFilesystem` filesystem object to use as the authoritative,
up-to-date source for the archive
cache : object
:py:mod:`pyFilesystem` filesystem object to use as the cache. Default
``None``.
use_cache : bool
update, service_path, version_check, \*\*kwargs
'''
if write_path is None:
write_path = read_path
with _choose_read_fs(
authority, cache, read_path, version_check, hasher) as read_fs:
write_mode = ('w' in mode) or ('a' in mode) or ('+' in mode)
if write_mode:
readwrite_mode = (
('a' in mode) or (
('r' in mode) and (
'+' in mode)))
with _prepare_write_fs(
read_fs, cache, read_path, readwrite_mode) as write_fs:
wrapper = MultiFS()
wrapper.addfs('reader', read_fs)
wrapper.setwritefs(write_fs)
with wrapper.open(read_path, mode, *args, **kwargs) as f:
yield f
info = write_fs.getinfokeys(read_path, 'size')
if 'size' in info:
if info['size'] == 0:
return
with write_fs.open(read_path, 'rb') as f:
checksum = hasher(f)
if not version_check(checksum):
if (
cache_on_write or
(
cache
and (
fs.path.abspath(read_path) ==
fs.path.abspath(write_path))
and cache.fs.isfile(read_path)
)
):
_makedirs(cache.fs, fs.path.dirname(write_path))
fs.utils.copyfile(
write_fs, read_path, cache.fs, write_path)
_makedirs(authority.fs, fs.path.dirname(write_path))
fs.utils.copyfile(
cache.fs, write_path, authority.fs, write_path)
else:
_makedirs(authority.fs, fs.path.dirname(write_path))
fs.utils.copyfile(
write_fs, read_path, authority.fs, write_path)
update(**checksum)
else:
with read_fs.open(read_path, mode, *args, **kwargs) as f:
yield f
