class InitialResourceAdditionsRole(Struct):
class UsernameStatement(Struct):
usernames = Field([str])
id = Field(str)
permissions = Field([str])
statements = Field([UnionType({"username": UsernameStatement})])
def execute(self):
logger = logging.getLogger(__name__)
resource = Resource.get(id=self.id)
if not resource:
logger.error(
'InitialResourceAdditionsRole resource {!r} does not exist'.
format(self.id))
return
if not resource.initial:
logger.error(
'InitialResourceAdditionsRole resource {!r} must be initial'.
format(self.id))
return
for statement in self.statements:
assert isinstance(statement, self.UsernameStatement)
for username in statement.usernames:
user = User.get(username=username)
if not user:
logger.error(
'InitialResourceAdditionsRole user {!r} does not exist'
.format(username))
continue
for perm in self.permissions:
Permission(name=perm, user=user, resource=resource)
class SmartProcessor(Struct):
"""
This processor picks the #GoogleProcessor, #SphinxProcessor or #PydocmdProcessor after
guessing which is appropriate from the syntax it finds in the docstring.
"""
google = Field(GoogleProcessor, default=GoogleProcessor)
pydocmd = Field(PydocmdProcessor, default=PydocmdProcessor)
sphinx = Field(SphinxProcessor, default=SphinxProcessor)
@override
def process(self, modules: List[docspec.Module],
resolver: Optional[Resolver]) -> None:
docspec.visit(modules, self._process)
def _process(self, obj: docspec.ApiObject):
if not obj.docstring:
return None
for name in ('google', 'pydocmd', 'sphinx'):
indicator = '@doc:fmt:' + name
if indicator in obj.docstring:
obj.docstring = obj.docstring.replace(indicator, '')
return getattr(self, name)._process(obj)
if self.sphinx.check_docstring_format(obj.docstring):
return self.sphinx._process(obj)
if self.google.check_docstring_format(obj.docstring):
return self.google._process(obj)
return self.pydocmd._process(obj)
class InitialUserRole(Struct):
username = Field(str)
password = Field(str, default=None)
password_hash = Field(str, JsonFieldName('password-hash'), default=None)
def execute(self):
logger = logging.getLogger(__name__)
if not self.password and not self.password_hash:
logger.warning('InitialUserRole missing password/password-hash')
return
user = User.get(username=self.username)
if user and not user.initial:
logger.error(
'InitialUserRole targets non-initial user {!r}'.format(
user.username))
return
if user:
if self.password:
user.set_password(self.password)
else:
user.password_hash = self.password_hash
logger.info('Updated initial user {!r}'.format(self.username))
else:
user = User(username=self.username,
password=self.password,
password_hash=self.password_hash,
initial=True)
logger.info('Created initial user {!r}'.format(self.username))
class WsgiAppConfig(Struct):
"""
Basic app configuration that is bassed to a #IWsgiRunner to run an application.
"""
entrypoint = Field(str)
bind = Field(BindConfig, default=lambda: BindConfig('127.0.0.1', 8000))
ssl = Field(SslConfig, default=None)
files = Field(FilesConfig, default=lambda: FilesConfig.from_dir('./var'))
runners = Field(dict(value_type=IWsgiRunner))
@classmethod
def load(self,
filename: str,
mapper: ObjectMapper = None) -> 'WsgiAppConfig':
"""
Loads the #WsgiAppConfig from a YAML file specified with *filename*.
"""
if not mapper:
mapper = ObjectMapper(JsonModule())
with open(filename) as fp:
return mapper.deserialize(yaml.safe_load(fp),
cls,
filename=filename)
def run(self, runner: str, daemonize: bool = False) -> None:
self.runners[runner].run(self, daemonize)
class DemistoMarkdownRenderer(MarkdownRenderer):
func_prefix = Field(str, default=None)
module_overview = Field(str, default=None)
# Overriding header levels from MarkdownRenderer to Function header to level 2
header_level_by_type = Field({int},
default={
'Module': 1,
'Class': 2,
'Method': 4,
'Function': 2,
'Data': 4,
})
def _format_function_signature(self,
func: docspec.Function,
override_name: str = None,
add_method_bar: bool = True) -> str:
function_signature = super()._format_function_signature(
func, override_name, add_method_bar)
if self.func_prefix:
function_signature = f'{self.func_prefix}{function_signature}'
return function_signature
def _render_header(self, fp, level, obj):
if isinstance(obj, docspec.Module) and self.module_overview:
fp.write(self.module_overview)
fp.write('\n\n')
else:
super()._render_header(fp, level, obj)
class ImageWithResolution(Struct): """ Represents an actual URL to an image and it's resolution. """ height = Field(int) width = Field(int) image_url = Field(str) filename = Field(str)
class BaseGitServiceSourceLinker(BaseGitSourceLinker):
#: The repository name, formatted as `owner/repo`.
repo = Field(str)
#: The Gitea host name. Defaults to `gitea.com`.
host = Field(str)
def get_context_vars(self) -> Dict[str, str]:
return {'repo': self.repo, 'host': self.host}
class Page(deconflicted_base(Struct, Generic)):
"""
Template that represents a paginated type for a specific item type. Can
be instantiated with the #of() method.
"""
items = Field([Any])
next_page_token = Field(Any, FieldName('nextPageToken'), nullable=True)
def __generic_init__(self, item_type: Any, token_type: Any = int):
self.items = Field([item_type])
self.next_page_token = Field(token_type, FieldName('nextPageToken'), nullable=True)
class SetNowPlayingRequest(Struct):
track_id = Field(int, JsonFieldName('trackId'))
status = Field(str)
PLAYING = 'playing'
PAUSED = 'paused'
STOPPED = 'stopped'
@JsonValidator
def validate(self):
if self.status not in (self.PLAYING, self.PAUSED, self.STOPPED):
raise ValueError('invalid status: {!r}'.format(self.status))
class ImageCredit(Struct): """ Data to credit the author or photographer of a wallpaper. """ #: Text that can be printed / displayed to credit the author. text = Field(str) #: Name of the author. author = Field(str) #: Url to the author's webpage. author_url = Field(str)
class BindConfig(Struct):
host = Field(str)
port = Field(int)
@classmethod
def __deserialize(cls, context, node):
if not isinstance(node.value, str):
raise NotImplementedError
host, port = node.value.partition(':')
try:
port = int(port)
except ValueError as exc:
raise node.value_error(exc)
return cls(host, port)
class HugoPage(Page): """ A subclass of #Page which adds Hugo-specific overrides. ### Options """ children = Field([HugoPage], default=list) #: The Hugo preamble of the page. This is merged with the #HugoRenderer.default_preamble. preamble = Field(dict, default=dict) #: Override the directory that this page is rendered into (relative to the #: content directory). Defaults to `null`. directory = Field(str, default=None)
class HugoThemeGitUrl(Struct):
clone_url = Field(str)
postinstall = Field([str], default=list)
@property
def name(self):
return posixpath.basename(self.clone_url).rstrip('.git').lstrip('hugo-')
def install(self, theme_dir: str):
dst = os.path.join(theme_dir, self.name)
if not os.path.isdir(dst):
command = ['git', 'clone', self.clone_url, dst, '--depth', '1',
'--recursive', '--shallow-submodules']
subprocess.check_call(command)
for command in self.postinstall:
subprocess.check_call(command, shell=True, cwd=dst)
class CustomizedMarkdownRenderer(MarkdownRenderer):
"""We override some defaults in this subclass. """
#: Disabled because Docusaurus supports this automatically.
insert_header_anchors = Field(bool, default=False)
#: Escape html in docstring, otherwise it could lead to invalid html.
escape_html_in_docstring = Field(bool, default=True)
#: Conforms to Docusaurus header format.
render_module_header_template = Field(
str,
default=('---\n'
'sidebar_label: {relative_module_name}\n'
'title: {module_name}\n'
'---\n\n'))
class VoteRequest(Struct):
vote = Field(str)
is_upvote = property(lambda self: self.vote == 'up')
@JsonValidator
def validate(self):
if self.vote not in ('up', 'down'):
raise ValueError('invalid value for "vote": {!r}'.format(self.vote))
class HugoConfig(Struct):
"""
Represents the Hugo configuration file that is rendered into the build directory.
### Options
"""
#: Base URL.
baseURL = Field(str, default=None)
#: Language code. Default: `en-us`
languageCode = Field(str, default='en-us')
#: Title of the site. This is a mandatory field.
title = Field(str)
#: The theme of the site. This is a mandatory field. It must be a string, a #HugoThemePath
#: or a #HugoThemeGitUrl object. Examples:
#:
#: ```yml
#: theme: antarctica
#: theme: {clone_url: "https://github.com/alex-shpak/hugo-book.git"}
#: theme: docs/hugo-theme/
#: ```
theme = Field((str, HugoThemePath, HugoThemeGitUrl))
#: This field collects all remaining options that do not match any of the above
#: and will be forwarded directly into the Hugo `config.yaml` when it is rendered
#: into the build directory.
additional_options = Field(dict, Remainder())
def to_toml(self, fp: TextIO) -> None:
data = self.additional_options.copy()
for field in self.__fields__:
if field in ('additional_options', 'theme'):
continue
value = getattr(self, field)
if value:
data[field] = value
if isinstance(self.theme, str):
data['theme'] = self.theme
elif isinstance(self.theme, (HugoThemePath, HugoThemeGitUrl)):
data['theme'] = self.theme.name
else:
assert False
fp.write(toml.dumps(data))
class Custom(Struct):
"""
Can be used to implement custom route parameters that are tied to a
specific framework and they cannot be properly supported in the client
generation without custom code additions.
"""
read = Field(Any) # type: Callable[[Any], Any]
class FilesConfig(Struct):
pidfile = Field(str, default=None)
stdout = Field(str, default=None)
stderr = Field(str, default=None)
@classmethod
def deserialize(cls, context, node) -> 'FilesConfig':
if not isinstance(node.value, str):
raise NotImplementedError
return cls.from_dir(node.value)
@classmethod
def from_dir(cls, dirpath: str) -> 'FilesConfig':
return cls(
os.path.join(dirpath, 'run', 'wsgi.pid'),
os.path.join(dirpath, 'log', 'stdout.log'),
os.path.join(dirpath, 'log', 'stderr.log'),
)
class BitbucketSourceLinker(BaseGitServiceSourceLinker):
"""
Source linker for Git repositories hosted on bitbucket.org or any self-hosted Bitbucket instance.
"""
host = Field(str, default="bitbucket.org")
def get_url_template(self) -> str:
return 'https://{host}/{repo}/src/{sha}/{path}#lines-{lineno}'
class GithubSourceLinker(BaseGitServiceSourceLinker):
"""
Source linker for Git repositories hosted on github.com or any self-hosted GitHub instance.
"""
host = Field(str, default="github.com")
def get_url_template(self) -> str:
return 'https://{host}/{repo}/blob/{sha}/{path}#L{lineno}'
class CredentialsConfig(Struct):
youtube_data_api = Field(str, JsonFieldName('youtube-data-api'))
def create_youtube_client(self):
from googleapiclient import discovery
from ..clients.youtube import Youtube
service = discovery.build('youtube',
'v3',
developerKey=self.youtube_data_api)
return Youtube(service)
class IterHierarchyItem(Struct):
page = Field(Page)
parent_chain = Field([Page])
def filename(
self,
parent_dir: Optional[str], #: Parent directory to join with
suffix_with_dot: str, #: Suffix of the generated filename
index_name: str = 'index',
skip_empty_pages: bool = True,
) -> Optional[str]:
path = [p.name for p in self.parent_chain] + [self.page.name]
if self.page.children:
if skip_empty_pages and not self.page.contents and not self.page.source:
return None
path.append(index_name)
filename = os.path.join(*path) + suffix_with_dot
if parent_dir:
filename = os.path.join(parent_dir, filename)
return filename
class GunicornRunner(Struct):
"""
Runs a WSGI application using [Gunicorn][0].
[0]: https://gunicorn.org/
"""
num_workers = Field(int, default=None)
additional_options = Field([str], default=list)
@override
def run(self, app_config: WsgiAppConfig, daemonize: bool = False) -> None:
command = [
'gunicorn', app_config.entrypoint, '--bind',
'{}:{}'.format(app_config.bind.host, app_config.bind.port)
]
if daemonize:
command.append('--daemon')
if app_config.files.pidfile:
os.makedirs(os.path.dirname(app_config.files.pidfile),
exist_ok=True)
command += ['--pid', app_config.files.pidfile]
if app_config.files.stdout:
os.makedirs(os.path.dirname(app_config.files.stdout),
exist_ok=True)
command += ['--access-logfile', app_config.files.stdout]
if app_config.files.stderr:
os.makedirs(os.path.dirname(app_config.files.stderr),
exist_ok=True)
command += ['--error-logfile', app_config.files.stderr]
if app_config.ssl:
command += [
'--certfile', app_config.ssl.cert, '--keyfile',
app_config.ssl.key
]
if self.num_workers:
command += ['--workers', str(self.num_workers)]
command += self.additional_options
env = os.environ.copy()
env['_WSGI_RUNNER_CLASS'] = _type_id(self)
subprocess.call(command, env=env)
class GitSourceLinker(BaseGitSourceLinker):
"""
This source linker allows you to define your own URL template to generate a permalink for
an API object.
"""
#: The URL template as a Python format string. Available variables are "path", "sha" and
#: "lineno".
url_template = Field(str)
def get_url_template(self) -> str:
return self.url_template
class HugoThemePath(Struct):
path = Field(str)
@property
def name(self):
return os.path.basename(self.path)
def install(self, themes_dir: str):
# TODO (@NiklasRosenstein): Support Windows (which does not support symlinking).
dst = os.path.join(self.name)
if not os.path.lexists(dst):
os.symlink(self.path, dst)
class WallpaperSpec(Struct):
"""
Represents the resolved data for a wallpaper.
"""
#: The name of the wallpaper.
name = Field(str)
#: A list of keywords for this wallpaper.
keywords = Field([str])
#: Source URL of this wallpaper. This should be the URL to a standard webpage that can
#: be viewed in the browser (not a JSON payload or just the raw image). Usually this
#: points at the main page on the wallpaper host.
source_url = Field(str)
#: Credit to the author/photographer for this wallpaper.
credit = Field(ImageCredit)
#: Available image resolutions, sorted from highest to lowest.
resolutions = Field([ImageWithResolution])
#: A mapping of the closest resolution alias to the image and its actual resolution.
#: Usually this is automatically generated from #resolutions, but it is stored in the
#: object as a cache.
resolution_aliases = Field(dict(value_type=ImageWithResolution), default=dict)
def normalize(self) -> None:
"""
Normalize the contents by sorting #resolutions and re-generating #resolution_aliases.
"""
self.resolutions.sort(key=lambda x: x.width * x.height, reverse=True)
self.resolution_aliases = {}
# Find the closest matching resolution alias for the available resolutions.
for alias, (width, height) in RESOLUTION_ALIASES.items():
for image in self.resolutions:
if image.width >= width and image.height >= height:
self.resolution_aliases[alias] = image
continue
def to_json(self, out: Union[TextIO, str, None], **kwargs) -> dict:
if isinstance(out, str):
with open(out, 'w') as fp:
return self.to_json(fp, **kwargs)
data = MAPPER.serialize(self, WallpaperSpec)
if out:
json.dump(data, out, **kwargs)
return data
@classmethod
def from_json(cls, in_: Union[TextIO, str, dict], filename: str = None) -> 'WallpaperSpec':
filename = filename or getattr(in_, 'name', None)
if isinstance(in_, str):
with open(in_) as fp:
return cls.from_json(fp)
elif hasattr(in_, 'read'):
in_ = json.load(in_)
return MAPPER.deserialize(in_, cls, filename=filename)
class RuntimeConfig(Struct):
credentials = Field(CredentialsConfig)
database = Field(dict)
external_url = Field(str, default=None)
frontend_url = Field(str, default=None)
produces = Field([dict])
server = Field(ServerConfig)
def get_external_url(self):
if self.external_url is None:
return 'http://{}:{}'.format(self.host, self.port)
return self.external_url
def get_frontend_url(self):
if self.frontend_url is None:
return self.get_external_url()
return self.frontend_url
class InitialResourceRole(Struct):
id = Field(str)
@classmethod
def execute_once(self):
logger = logging.getLogger(__name__)
logger.info('Bulk-deleting initial resources')
Resource.select(lambda r: r.initial).delete(bulk=True)
def execute(self):
logger = logging.getLogger(__name__)
resource = Resource.get(id=self.id)
if resource is not None:
if resource.initial:
logger.warn('InitialResourceRole {!r} already exists'.format(
self.id))
else:
logger.error(
'InitialResourceRole {!r} is a non-initial resource'.
format(self.id))
return
resource = Resource(id=self.id, initial=True)
logger.info('InitialResourceRole {!r} created'.format(self.id))
class PythonLoader(Struct):
"""
This implementation of the #Loader interface parses Python modules and packages using
#docspec_python. See the options below to control which modules and packages are being
loaded and how to configure the parser.
With no #modules or #packages set, the #PythonLoader will discover available modules
in the current and `src/` directory.
__lib2to3 Quirks__
Pydoc-Markdown doesn't execute your Python code but instead relies on the
`lib2to3` parser. This means it also inherits any quirks of `lib2to3`.
_List of known quirks_
* A function argument in Python 3 cannot be called `print` even though
it is legal syntax
"""
#: A list of module names that this loader will search for and then parse.
#: The modules are searched using the #sys.path of the current Python
# interpreter, unless the #search_path option is specified.
modules = Field([str], default=None)
#: A list of package names that this loader will search for and then parse,
#: including all sub-packages and modules.
packages = Field([str], default=None)
#: The module search path. If not specified, the current #sys.path is
#: used instead. If any of the elements contain a `*` (star) symbol, it
#: will be expanded with #sys.path.
search_path = Field([str], default=None)
#: List of modules to ignore when using module discovery on the #search_path.
ignore_when_discovered = Field([str], default=lambda: ['test', 'tests', 'setup'])
#: Options for the Python parser.
parser = Field(docspec_python.ParserOptions, default=Field.DEFAULT_CONSTRUCT)
#: The encoding to use when reading the Python source files.
encoding = Field(str, default=None)
_context = Field(Context, default=None, hidden=True)
def get_effective_search_path(self) -> List[str]:
if self.search_path is None:
search_path = ['.', 'src'] if self.modules is None else list(sys.path)
else:
search_path = list(self.search_path)
if '*' in search_path:
index = search_path.index('*')
search_path[index:index+1] = sys.path
return [os.path.join(self._context.directory, x) for x in search_path]
# Loader
@override
def load(self) -> Iterable[docspec.Module]:
search_path = self.get_effective_search_path()
modules = list(self.modules or [])
packages = list(self.packages or [])
do_discover = (self.modules is None and self.packages is None)
if do_discover:
for path in search_path:
try:
discovered_items = list(docspec_python.discover(path))
except FileNotFoundError:
continue
logger.info(
'Discovered Python modules %s and packages %s in %r.',
[x.name for x in discovered_items if x.is_module()],
[x.name for x in discovered_items if x.is_package()],
path,
)
for item in discovered_items:
if item.name in self.ignore_when_discovered:
continue
if item.is_module():
modules.append(item.name)
elif item.is_package():
packages.append(item.name)
logger.info(
'Load Python modules (search_path: %r, modules: %r, packages: %r, discover: %s)',
search_path, modules, packages, do_discover
)
return docspec_python.load_python_modules(
modules=modules,
packages=packages,
search_path=search_path,
options=self.parser,
encoding=self.encoding,
)
# PluginBase
@override
def init(self, context: Context) -> None:
self._context = context
class PydocMarkdown(Struct):
"""
This object represents the main configuration for Pydoc-Markdown.
"""
#: A list of loader implementations that load #docspec.Module#s.
#: Defaults to #PythonLoader.
loaders = Field([Loader], default=lambda: [PythonLoader()])
#: A list of processor implementations that modify #docspec.Module#s. Defaults
#: to #FilterProcessor, #SmartProcessor and #CrossrefProcessor.
processors = Field([Processor], default=lambda: [
FilterProcessor(), SmartProcessor(), CrossrefProcessor()])
#: A renderer for #docspec.Module#s. Defaults to #MarkdownRenderer.
renderer = Field(Renderer, default=MarkdownRenderer)
#: Hooks that can be executed at certain points in the pipeline. The commands
#: are executed with the current `SHELL`.
hooks = Field({
'pre_render': Field([str], FieldName('pre-render'), default=list),
'post_render': Field([str], FieldName('post-render'), default=list),
}, default=Field.DEFAULT_CONSTRUCT)
# Hidden fields are filled at a later point in time and are not (de-) serialized.
unknown_fields = Field([str], default=list, hidden=True)
def __init__(self, *args, **kwargs) -> None:
super(PydocMarkdown, self).__init__(*args, **kwargs)
self.resolver: Optional[Resolver] = None
self._context: Optional[Context] = None
def load_config(self, data: Union[str, dict]) -> None:
"""
Loads a YAML configuration from *data*.
:param data: Nested structurre or the path to a YAML configuration file.
"""
filename = None
if isinstance(data, str):
filename = data
logger.info('Loading configuration file "%s".', filename)
data = ytemplate.load(filename, {'env': ytemplate.Attributor(os.environ)})
collector = Collect()
result = mapper.deserialize(data, type(self), filename=filename, decorations=[collector])
vars(self).update(vars(result))
self.unknown_fields = list(concat((str(n.locator.append(u)) for u in n.unknowns)
for n in collector.nodes))
def init(self, context: Context) -> None:
"""
Initialize all plugins with the specified *context*. Cannot be called multiple times.
If omitted, the plugins will be initialized with a default context before the load,
process or render phase.
"""
if self._context:
raise RuntimeError('already initialized')
self._context = context
logger.debug('Initializing plugins with context %r', context)
for loader in self.loaders:
loader.init(context)
for processor in self.processors:
processor.init(context)
self.renderer.init(context)
def ensure_initialized(self) -> None:
if not self._context:
self.init(Context(directory='.'))
def load_modules(self) -> List[docspec.Module]:
"""
Loads modules via the #loaders.
"""
logger.info('Loading modules.')
self.ensure_initialized()
modules = []
for loader in self.loaders:
modules.extend(loader.load())
return modules
def process(self, modules: List[docspec.Module]) -> None:
"""
Process modules via the #processors.
"""
self.ensure_initialized()
if self.resolver is None:
self.resolver = self.renderer.get_resolver(modules)
for processor in self.processors:
processor.process(modules, self.resolver)
def render(self, modules: List[docspec.Module], run_hooks: bool = True) -> None:
"""
Render modules via the #renderer.
"""
self.ensure_initialized()
if run_hooks:
self.run_hooks('pre-render')
if self.resolver is None:
self.resolver = self.renderer.get_resolver(modules)
self.renderer.process(modules, self.resolver)
self.renderer.render(modules)
if run_hooks:
self.run_hooks('post-render')
def build(self, site_dir: str) -> None:
if not Builder.provided_by(self.renderer):
name = type(self.renderer).__name__
raise NotImplementedError('Renderer "{}" does not support building'.format(name))
self.ensure_initialized()
self.renderer.build(site_dir)
def run_hooks(self, hook_name: str) -> None:
assert self._context is not None
for command in getattr(self.hooks, hook_name.replace('-', '_')):
subprocess.check_call(command, shell=True, cwd=self._context.directory)
