def _update_themes():
import subprocess
import semaver
from os import path
from pytsite import console, lang, pip, plugman
from plugins import assetman
from . import _api
for theme in _api.get_all().values():
# Update theme from git repository
if path.exists(path.join(theme.path, '.git')):
console.print_info(
lang.t('theming@updating_theme', {'name': theme.name}))
subprocess.call(['git', '-C', theme.path, 'pull'])
console.print_info(
lang.t('theming@installing_theme_requirements',
{'name': theme.name}))
# Install/upgrade required pip packagers
for p_name, p_ver in theme.requires['packages'].items():
pip.install(p_name, p_ver, True)
# Install or update required plugins
for p_name, p_ver in theme.requires['plugins'].items():
plugman.install(p_name, semaver.VersionRange(p_ver))
# Compile theme assets
if assetman.is_package_registered(theme.package_name):
assetman.build(theme.package_name)
def update_stage_2():
# Install/update pip packages
_console.print_info(_lang.t('pytsite.pip@updating_packages'))
for pkg_name, pkg_ver in _package_info.requires_packages('app').items():
try:
_api.install(pkg_name, pkg_ver, True, _reg.get('debug'))
except _error.PackageInstallError as e:
raise _console.error.CommandExecutionError(e)
def _subprocess_run(cmd: list):
r = _subprocess.run(cmd, stdout=_subprocess.PIPE, stderr=_subprocess.PIPE)
if r.returncode != 0:
_console.print_warning(r.stderr.decode('utf-8'))
if _DEBUG and r.stdout:
_console.print_info(r.stdout.decode('utf-8'))
def exec(self):
if not self.args:
raise _console.error.MissingArgument('pytsite.pip@package_name_not_specified')
try:
for pkg_name in self.args:
_console.print_info(_pip.uninstall(pkg_name))
except _pip.error.Error as e:
raise _console.error.CommandExecutionError(e)
def reindex(model: str = None):
"""Reindex model(s)'s collection
"""
if model:
console.print_info(lang.t('odm@reindex_model', {'model': model}))
dispense(model).reindex()
else:
for model in get_registered_models():
reindex(model)
def _do_reload():
"""Modify 'touch.reload' file
"""
touch_reload_path = _path.join(_reg.get('paths.storage'), 'touch.reload')
_events.fire('pytsite.reload@before_reload')
with open(touch_reload_path, 'wt') as f:
f.write(_util.w3c_datetime_str())
_events.fire('pytsite.reload@reload')
_console.print_info(_lang.t('pytsite.reload@app_is_reloading'))
def exec(self):
if not self.args:
return
try:
maintenance.enable()
console.print_info(
lang.t('assetman@installing_npm_packages',
{'packages': ', '.join(self.args)}))
_api.npm_install(self.args)
except RuntimeError as e:
raise console.error.CommandExecutionError(e)
finally:
maintenance.disable()
def exec(self):
if self.args:
packages = {}
for pkg_spec in self.args:
match = _PKG_SPEC_RE.findall(pkg_spec)
if not match:
raise _console.error.CommandExecutionError('Invalid package identifier: {}'.format(pkg_spec))
packages[match[0][0]] = match[0][1]
else:
packages = _package_info.requires_packages('app').items()
for pkg_name, pkg_version in packages.items():
pkg_spec = '{}{}'.format(pkg_name, pkg_version)
try:
_console.print_info(_lang.t('pytsite.pip@package_installing', {'package': pkg_spec}))
_pip.install(pkg_name, pkg_version, self.opt('upgrade'), self.opt('debug'))
_console.print_success(_lang.t('pytsite.pip@package_successfully_installed', {'package': pkg_spec}))
except _pip.error.Error as e:
raise _console.error.CommandExecutionError(e)
def build_all(debug: bool = _DEBUG, mode: str = None):
console.print_info(lang.t('assetman@compiling_assets'))
assets_static_path = reg.get('paths.assets')
node_modules_dir = path.join(reg.get('paths.root'), 'node_modules')
node_modules_subdir = path.join(node_modules_dir, '@pytsite')
if not path.isdir(node_modules_dir):
raise FileNotFoundError(
"'{}' directory is not exists. Check your NPM installation.")
if not path.isdir(node_modules_subdir):
mkdir(node_modules_subdir, 0o755)
if path.exists(assets_static_path):
rmtree(assets_static_path)
for pkg_name in _packages:
build(pkg_name, debug, mode)
def on_register_storage_driver(driver: _driver.Storage):
# Create/update minimum set of built-in roles
for name in ('anonymous', 'user', 'admin', 'dev'):
try:
# Check for role and its valid description
role = _api.get_storage_driver().get_role(name)
valid_desc = 'auth@{}_role_description'.format(name)
if role.description != valid_desc:
_api.switch_user_to_system()
role.description = valid_desc
role.save()
_api.restore_user()
except _error.RoleNotFound:
# Create role
_api.get_storage_driver().create_role(
name, 'auth@{}_role_description'.format(name)).save()
console.print_info(lang.t('auth@role_created', {'name': name}))
# Switch user context
if reg.get('env.type') == 'console':
_api.switch_user_to_system()
def plugin_update(v_from: _Version):
# Field 'uid' added to users and roles
if v_from <= '2.3':
from pytsite import console, mongodb
from plugins import odm
for c in ('users', 'roles'):
col = mongodb.get_collection(c)
for d in col.find():
col.update_one({'_id': d['_id']}, {'$set': {'uid': str(d['_id'])}})
console.print_info('Document updated: {}:{}'.format(c, d['_id']))
odm.clear_cache('role')
odm.clear_cache('user')
odm.reindex('role')
odm.reindex('user')
if v_from <= '3.2':
import re
from pytsite import console, mongodb
from plugins import odm
for c in ('users', 'roles'):
col = mongodb.get_collection(c)
for d in col.find():
col.update_one({'_id': d['_id']}, {'$set': {'uid': d['_ref']}})
console.print_info('Document updated: {}:{}'.format(c, d['_id']))
odm.clear_cache('role')
odm.clear_cache('user')
odm.reindex('role')
odm.reindex('user')
db_obj_id_re = re.compile('[a-z:]+([0-9a-f]{24})$')
for m in odm.get_registered_models():
mock = odm.dispense(m)
for f_name, f in mock.fields.items():
for d in mock.collection.find():
f_new_value = None
if isinstance(f, field.User) and d[f_name]:
f_new_value = '{}:{}'.format('user', db_obj_id_re.sub('\\1', d[f_name]))
if isinstance(f, (field.Users, field.Roles)) and d[f_name]:
auth_model = 'role' if isinstance(f, field.Roles) else 'user'
f_new_value = ['{}:{}'.format(auth_model, db_obj_id_re.sub('\\1', v)) for v in d[f_name]]
if f_new_value:
mock.collection.update_one({'_id': d['_id']}, {'$set': {f_name: f_new_value}})
console.print_info('Document updated: {}:{}'.format(m, d['_id']))
odm.clear_cache(m)
if v_from <= '3.4':
from plugins import odm
odm.reindex('role')
odm.reindex('user')
def setup():
"""Setup assetman environment
"""
console.print_info(lang.t('assetman@installing_required_npm_packages'))
root_dir = reg.get('paths.root')
node_modules_subdir = path.join(path.join(root_dir, 'node_modules'),
'@pytsite')
dev_host_npm_packages_dir = path.join(root_dir, 'npm_packages')
is_dev_host = path.isdir(dev_host_npm_packages_dir)
makedirs(node_modules_subdir, 0o755, True)
# Create symlinks in node_modules from npm_packages
if is_dev_host:
for name in listdir(dev_host_npm_packages_dir):
src = path.join(dev_host_npm_packages_dir, name)
if path.isdir(src):
dst = path.join(node_modules_subdir, name)
if not path.exists(dst):
symlink(src, dst)
# Create symlinks in node_modules from registered packages which have package.json
for pkg_name in _packages:
src_dir = assets_src(pkg_name)
if not path.exists(path.join(src_dir, 'package.json')):
continue
node_pkg_name = pkg_name.replace('plugins.',
'').replace('_',
'-').replace('.', '-')
node_modules_pkg_dir = path.join(node_modules_subdir, node_pkg_name)
if not path.exists(node_modules_pkg_dir):
symlink(src_dir, node_modules_pkg_dir)
# Install NPM packages required by plugins
install_npm_deps(list(_packages.keys()))
def exec(self):
"""Execute The Command.
"""
maint = not self.opt('no-maint')
debug = self.opt('debug')
mode = self.opt('mode')
watch = self.opt('watch')
if watch and not self.args:
raise console.error.CommandExecutionError(
'--watch option must be used only with package name')
try:
if maint and not watch:
maintenance.enable()
packages = self.args
if packages:
if len(packages) > 1:
watch = False
for package in packages:
_api.build(package, debug, mode, watch)
else:
_api.build_all(debug, mode)
except (RuntimeError, _error.PackageNotRegistered,
_error.PackageAlreadyRegistered) as e:
raise console.error.CommandExecutionError(e)
except KeyboardInterrupt:
console.print_info('')
finally:
if maint and not watch:
maintenance.disable()
def plugin_update(v_from: _Version):
from pytsite import console
if v_from < '1.4':
from pytsite import mongodb
for collection_name in mongodb.get_collection_names():
collection = mongodb.get_collection(collection_name)
for doc in collection.find():
if '_ref' in doc:
continue
doc['_ref'] = '{}:{}'.format(doc['_model'], doc['_id'])
collection.replace_one({'_id': doc['_id']}, doc)
console.print_info('Document {} updated'.format(doc['_ref']))
if v_from < '1.6':
console.run_command('odm:reindex')
if v_from < '4.0':
# Update all entities that have `Ref` and `RefsList` fields
for m in get_registered_models():
console.print_info("Processing model '{}'".format(m))
fields_to_update = []
for f in dispense(m).fields.values():
if f.name != '_parent' and isinstance(
f, (field.Ref, field.RefsList)):
fields_to_update.append(f.name)
if fields_to_update:
for e in find(m).get():
for f_name in fields_to_update:
e.f_set(f_name, e.f_get(f_name))
e.save(update_timestamp=False)
console.print_info('Entity {} updated, fields {}'.format(
e, fields_to_update))
if v_from < '6.0':
console.run_command('odm:reindex')
def db_restore():
for model in _api.get_registered_models():
_api.clear_cache(model)
console.print_info(lang.t('odm@cache_cleared', {'model': model}))
def exec(self):
"""Execute teh command
"""
model = self.arg(0)
# Checking if the content model registered
if not _api.is_model_registered(model):
raise console.error.CommandExecutionError("'{}' is not a registered content model".format(model))
author_login = self.opt('author')
num = self.opt('num')
images_num = self.opt('images')
language = self.opt('lang')
no_html = self.opt('no-html')
short = self.opt('short')
title_len = self.opt('title-len')
description_len = self.opt('description-len')
if no_html:
self.lorem_txt_args['format'] = 'text'
if short:
self.lorem_txt_args['paras'] = 1
users = list(auth.find_users(query.Query(query.Eq('status', 'active')), limit=10))
# Generate content entities
for m in range(0, num):
entity = _api.dispense(model)
# Author
if entity.has_field('author'):
if author_login:
author = auth.get_user(author_login)
if not author:
raise console.error.CommandExecutionError("'{}' is not a registered user".format(author_login))
else:
if not users:
raise console.error.CommandExecutionError(lang.t('content@no_users_found'))
rand = randint(0, len(users) - 1)
author = users[rand:rand + 1][0]
entity.f_set('author', author.uid)
# Title
if entity.has_field('title'):
entity.f_set('title', self._generate_title(title_len))
# Description
if entity.has_field('description'):
entity.f_set('description', self._generate_title(description_len))
# Body
if entity.has_field('body'):
body = []
for n in range(1, (images_num or 1) + 1):
body.append(requests.get(self.lorem_txt_url, self.lorem_txt_args).content.decode('utf-8'))
if not no_html and n > 1:
body.append('\n<p>[img:{}]</p>\n'.format(n))
entity.f_set('body', ''.join(body))
# Images
if entity.has_field('images') and images_num:
for n in range(0, images_num):
entity.f_add('images', file.create(self.lorem_img_url.format(randint(0, 1000))))
# Language
if entity.has_field('language'):
entity.f_set('language', language)
# Status
if entity.has_field('status'):
entity.f_set('status', CONTENT_STATUS_PUBLISHED)
events.fire('content@generate', entity=entity)
entity.save()
console.print_info(lang.t('content@new_content_created', {'model': entity.model, 'title': entity.title}))
def build(pkg_name: str,
debug: bool = _DEBUG,
mode: str = None,
watch: bool = False):
"""Compile assets
"""
pkg_name = resolve_package(pkg_name)
src = assets_src(pkg_name)
dst = assets_dst(pkg_name)
public_path = assets_public_path(pkg_name)
timestamps_path = path.join(assets_dst('assetman'), 'timestamps.json')
mode = mode or ('development' if debug else 'production')
# Build translations
if lang.is_package_registered(pkg_name):
build_translations(pkg_name)
# Building is possible only if 'webpack.config.js' exists
webpack_config = path.join(src, 'webpack.config.js')
if not path.exists(webpack_config):
return
# Clear destination directory
if path.exists(dst):
rmtree(dst)
# Create output directory for timestamps file
timestamps_dir_path = path.dirname(timestamps_path)
if not path.exists(timestamps_dir_path):
makedirs(timestamps_dir_path, 0o755, True)
# It is important timestamps file to be exists before start building process
if not path.isfile(timestamps_path):
with open(timestamps_path, 'wt', encoding='utf-8') as f:
f.write(json.dumps({}))
# Collect webpack's config parts from all the packages
webpack_parts = []
root_dir = reg.get('paths.root') + '/'
for p in _packages.values():
if path.exists(path.join(p[0], 'webpack.part.js')):
webpack_parts.append(p[0].replace(root_dir, ''))
# Run webpack
console.print_info(
lang.t('assetman@compiling_assets_for_package', {'package': pkg_name}))
args = [
'--mode',
mode,
'--config',
webpack_config,
'--context',
assets_src(pkg_name),
'--output-path',
dst,
'--output-public-path',
public_path,
'--env.NODE_ENV',
mode,
'--env.root_dir',
root_dir,
'--env.config_parts',
','.join(webpack_parts),
'--watch',
str(watch).lower(),
]
_run_node_bin('webpack-cli', args, watch or debug)
# Update timestamps
_BUILD_TS.put(pkg_name, int(time.time()))
with open(timestamps_path, 'wt', encoding='utf-8') as f:
logger.debug(f"Writing timestamps into '{timestamps_path}'")
f.write(json.dumps({k: _BUILD_TS.get(k) for k in _BUILD_TS.keys()}))
def on_pytsite_load():
update_info = _api.get_update_info()
if not update_info:
return
# If there waiting updates exist, reload the application
if _reg.get('env.type') == 'wsgi':
_logger.warn('Application needs to be loaded in console to finish plugins update')
return
failed_plugins = []
# Call 'plugin_pre_install()' hooks
for p_name, info in update_info.items():
v_to = _semver.Version(info['version_to'])
try:
# Check if the plugin is installed and loaded
plugin = _api.get(p_name)
# Call plugin_pre_install() hook
if hasattr(plugin, 'plugin_pre_install') and callable(plugin.plugin_pre_install):
plugin.plugin_pre_install()
# Fire 'pre_install' event
_events.fire('pytsite.plugman@pre_install', name=p_name, version=v_to)
except _error.PluginNotLoaded as e:
_logger.error(e)
_console.print_warning(_lang.t('pytsite.plugman@plugin_install_error', {
'plugin': p_name,
'version': v_to,
'msg': str(e),
}))
failed_plugins.append(p_name)
continue
# Finish installing/updating plugins
for p_name, info in update_info.items():
if p_name in failed_plugins:
continue
plugin = _api.get(p_name)
v_from = _semver.Version(info['version_from'])
v_to = _semver.Version(info['version_to'])
try:
_logger.info(_lang.t('pytsite.plugman@installing_plugin', {
'plugin': p_name,
'version': v_to,
}))
# Call plugin_install() hook
if hasattr(plugin, 'plugin_install') and callable(plugin.plugin_install):
plugin.plugin_install()
# Fire 'install' event
_events.fire('pytsite.plugman@install', name=p_name, version=v_to)
_console.print_success(_lang.t('pytsite.plugman@plugin_install_success', {
'plugin': p_name,
'version': v_to,
}))
except Exception as e:
_logger.error(e)
_console.print_warning(_lang.t('pytsite.plugman@plugin_install_error', {
'plugin': p_name,
'version': v_to,
'msg': str(e),
}))
continue
# Update plugin
if v_from != '0.0.0':
try:
_console.print_info(_lang.t('pytsite.plugman@updating_plugin', {
'plugin': p_name,
'v_from': v_from,
'v_to': v_to,
}))
# Call plugin_update() hook
if hasattr(plugin, 'plugin_update') and callable(plugin.plugin_update):
plugin.plugin_update(v_from=v_from)
# Fire 'update' event
_events.fire('pytsite.plugman@update', name=p_name, v_from=v_from)
_console.print_success(_lang.t('pytsite.plugman@plugin_update_success', {
'plugin': p_name,
'version': v_to,
}))
except Exception as e:
_console.print_warning(_lang.t('pytsite.plugman@plugin_update_error', {
'plugin': p_name,
'version': v_to,
'msg': str(e),
}))
continue
# Remove info from update queue
_api.rm_update_info(p_name)
def on_pytsite_update_stage_2():
# Update all installed plugins
_console.print_info(_lang.t('pytsite.plugman@updating_plugins'))
_console.run_command('plugman:install', {'reload': False})
def exec(self, args: tuple = (), **kwargs):
"""Execute the command.
"""
app_path = _reg.get('paths.app')
config_path = _reg.get('paths.config')
stage = self.opt('stage')
stop_after = self.opt('stop-after')
_chdir(app_path)
_maintenance.enable()
d = self._get_update_data()
if not d['update_in_progress']:
d['pytsite_version_from'] = str(_package_info.version('pytsite'))
d['app_version_from'] = str(_package_info.version('app'))
d['update_in_progress'] = True
self._set_update_data(d)
# Stage 1: update pip and PytSite
if stage == 1:
_console.print_info(_lang.t('pytsite.update@updating_environment'))
# Update pip
_pip.install('pip', None, True, self.opt('debug'))
# Update PytSite
_pip.install('pytsite', _package_info.requires_pytsite('app'), True, self.opt('debug'))
d['pytsite_version_to'] = str(_package_info.version('pytsite', False))
self._set_update_data(d)
# Notify listeners
_events.fire('pytsite.update@stage_1')
if stop_after != 1:
_subprocess.call(['./console', 'update', '--stage=2'])
else:
_maintenance.disable()
# Stage 2: update application and configuration
elif stage == 2:
# Notify listeners about PytSite update
_events.fire('pytsite.update@pytsite', v_from=_semver.Version(d['pytsite_version_from']))
# Update configuration
if _path.exists(_path.join(config_path, '.git')):
_console.print_info(_lang.t('pytsite.update@updating_configuration'))
_subprocess_run(['git', '-C', config_path, 'pull'])
# Update application
if _path.exists(_path.join(app_path, '.git')):
_console.print_info(_lang.t('pytsite.update@updating_application'))
_subprocess_run(['git', '-C', app_path, 'pull'])
d['app_version_to'] = str(_package_info.version('app', False))
self._set_update_data(d)
# Notify listeners
_events.fire('pytsite.update@stage_2')
if stop_after != 2:
_subprocess.call(['./console', 'update', '--stage=3'])
else:
_maintenance.disable()
# Stage 3: finish update process
elif stage == 3:
_console.print_info(_lang.t('pytsite.update@applying_updates'))
# Notify listeners about application update
_events.fire('pytsite.update@app', v_from=_semver.Version(d['app_version_from']))
# Notify listeners
_events.fire('pytsite.update@update')
# Application's update hook
import app
if hasattr(app, 'app_update') and callable(app.app_update):
app.app_update(v_from=_semver.Version(d['app_version_from']))
# Mark that update was finished successfully
d['update_in_progress'] = False
self._set_update_data(d)
# Disable maintenance mode
_maintenance.disable()
# Reload the application
_reload.reload()
