def _update_themes(): import subprocess import semaver from os import path from pytsite import console, lang, pip, plugman from plugins import assetman from . import _api for theme in _api.get_all().values(): # Update theme from git repository if path.exists(path.join(theme.path, '.git')): console.print_info( lang.t('theming@updating_theme', {'name': theme.name})) subprocess.call(['git', '-C', theme.path, 'pull']) console.print_info( lang.t('theming@installing_theme_requirements', {'name': theme.name})) # Install/upgrade required pip packagers for p_name, p_ver in theme.requires['packages'].items(): pip.install(p_name, p_ver, True) # Install or update required plugins for p_name, p_ver in theme.requires['plugins'].items(): plugman.install(p_name, semaver.VersionRange(p_ver)) # Compile theme assets if assetman.is_package_registered(theme.package_name): assetman.build(theme.package_name)
def update_stage_2(): # Install/update pip packages _console.print_info(_lang.t('pytsite.pip@updating_packages')) for pkg_name, pkg_ver in _package_info.requires_packages('app').items(): try: _api.install(pkg_name, pkg_ver, True, _reg.get('debug')) except _error.PackageInstallError as e: raise _console.error.CommandExecutionError(e)
def _subprocess_run(cmd: list): r = _subprocess.run(cmd, stdout=_subprocess.PIPE, stderr=_subprocess.PIPE) if r.returncode != 0: _console.print_warning(r.stderr.decode('utf-8')) if _DEBUG and r.stdout: _console.print_info(r.stdout.decode('utf-8'))
def exec(self): if not self.args: raise _console.error.MissingArgument('pytsite.pip@package_name_not_specified') try: for pkg_name in self.args: _console.print_info(_pip.uninstall(pkg_name)) except _pip.error.Error as e: raise _console.error.CommandExecutionError(e)
def reindex(model: str = None): """Reindex model(s)'s collection """ if model: console.print_info(lang.t('odm@reindex_model', {'model': model})) dispense(model).reindex() else: for model in get_registered_models(): reindex(model)
def _do_reload(): """Modify 'touch.reload' file """ touch_reload_path = _path.join(_reg.get('paths.storage'), 'touch.reload') _events.fire('pytsite.reload@before_reload') with open(touch_reload_path, 'wt') as f: f.write(_util.w3c_datetime_str()) _events.fire('pytsite.reload@reload') _console.print_info(_lang.t('pytsite.reload@app_is_reloading'))
def exec(self): if not self.args: return try: maintenance.enable() console.print_info( lang.t('assetman@installing_npm_packages', {'packages': ', '.join(self.args)})) _api.npm_install(self.args) except RuntimeError as e: raise console.error.CommandExecutionError(e) finally: maintenance.disable()
def exec(self): if self.args: packages = {} for pkg_spec in self.args: match = _PKG_SPEC_RE.findall(pkg_spec) if not match: raise _console.error.CommandExecutionError('Invalid package identifier: {}'.format(pkg_spec)) packages[match[0][0]] = match[0][1] else: packages = _package_info.requires_packages('app').items() for pkg_name, pkg_version in packages.items(): pkg_spec = '{}{}'.format(pkg_name, pkg_version) try: _console.print_info(_lang.t('pytsite.pip@package_installing', {'package': pkg_spec})) _pip.install(pkg_name, pkg_version, self.opt('upgrade'), self.opt('debug')) _console.print_success(_lang.t('pytsite.pip@package_successfully_installed', {'package': pkg_spec})) except _pip.error.Error as e: raise _console.error.CommandExecutionError(e)
def build_all(debug: bool = _DEBUG, mode: str = None): console.print_info(lang.t('assetman@compiling_assets')) assets_static_path = reg.get('paths.assets') node_modules_dir = path.join(reg.get('paths.root'), 'node_modules') node_modules_subdir = path.join(node_modules_dir, '@pytsite') if not path.isdir(node_modules_dir): raise FileNotFoundError( "'{}' directory is not exists. Check your NPM installation.") if not path.isdir(node_modules_subdir): mkdir(node_modules_subdir, 0o755) if path.exists(assets_static_path): rmtree(assets_static_path) for pkg_name in _packages: build(pkg_name, debug, mode)
def on_register_storage_driver(driver: _driver.Storage): # Create/update minimum set of built-in roles for name in ('anonymous', 'user', 'admin', 'dev'): try: # Check for role and its valid description role = _api.get_storage_driver().get_role(name) valid_desc = 'auth@{}_role_description'.format(name) if role.description != valid_desc: _api.switch_user_to_system() role.description = valid_desc role.save() _api.restore_user() except _error.RoleNotFound: # Create role _api.get_storage_driver().create_role( name, 'auth@{}_role_description'.format(name)).save() console.print_info(lang.t('auth@role_created', {'name': name})) # Switch user context if reg.get('env.type') == 'console': _api.switch_user_to_system()
def plugin_update(v_from: _Version): # Field 'uid' added to users and roles if v_from <= '2.3': from pytsite import console, mongodb from plugins import odm for c in ('users', 'roles'): col = mongodb.get_collection(c) for d in col.find(): col.update_one({'_id': d['_id']}, {'$set': {'uid': str(d['_id'])}}) console.print_info('Document updated: {}:{}'.format(c, d['_id'])) odm.clear_cache('role') odm.clear_cache('user') odm.reindex('role') odm.reindex('user') if v_from <= '3.2': import re from pytsite import console, mongodb from plugins import odm for c in ('users', 'roles'): col = mongodb.get_collection(c) for d in col.find(): col.update_one({'_id': d['_id']}, {'$set': {'uid': d['_ref']}}) console.print_info('Document updated: {}:{}'.format(c, d['_id'])) odm.clear_cache('role') odm.clear_cache('user') odm.reindex('role') odm.reindex('user') db_obj_id_re = re.compile('[a-z:]+([0-9a-f]{24})$') for m in odm.get_registered_models(): mock = odm.dispense(m) for f_name, f in mock.fields.items(): for d in mock.collection.find(): f_new_value = None if isinstance(f, field.User) and d[f_name]: f_new_value = '{}:{}'.format('user', db_obj_id_re.sub('\\1', d[f_name])) if isinstance(f, (field.Users, field.Roles)) and d[f_name]: auth_model = 'role' if isinstance(f, field.Roles) else 'user' f_new_value = ['{}:{}'.format(auth_model, db_obj_id_re.sub('\\1', v)) for v in d[f_name]] if f_new_value: mock.collection.update_one({'_id': d['_id']}, {'$set': {f_name: f_new_value}}) console.print_info('Document updated: {}:{}'.format(m, d['_id'])) odm.clear_cache(m) if v_from <= '3.4': from plugins import odm odm.reindex('role') odm.reindex('user')
def setup(): """Setup assetman environment """ console.print_info(lang.t('assetman@installing_required_npm_packages')) root_dir = reg.get('paths.root') node_modules_subdir = path.join(path.join(root_dir, 'node_modules'), '@pytsite') dev_host_npm_packages_dir = path.join(root_dir, 'npm_packages') is_dev_host = path.isdir(dev_host_npm_packages_dir) makedirs(node_modules_subdir, 0o755, True) # Create symlinks in node_modules from npm_packages if is_dev_host: for name in listdir(dev_host_npm_packages_dir): src = path.join(dev_host_npm_packages_dir, name) if path.isdir(src): dst = path.join(node_modules_subdir, name) if not path.exists(dst): symlink(src, dst) # Create symlinks in node_modules from registered packages which have package.json for pkg_name in _packages: src_dir = assets_src(pkg_name) if not path.exists(path.join(src_dir, 'package.json')): continue node_pkg_name = pkg_name.replace('plugins.', '').replace('_', '-').replace('.', '-') node_modules_pkg_dir = path.join(node_modules_subdir, node_pkg_name) if not path.exists(node_modules_pkg_dir): symlink(src_dir, node_modules_pkg_dir) # Install NPM packages required by plugins install_npm_deps(list(_packages.keys()))
def exec(self): """Execute The Command. """ maint = not self.opt('no-maint') debug = self.opt('debug') mode = self.opt('mode') watch = self.opt('watch') if watch and not self.args: raise console.error.CommandExecutionError( '--watch option must be used only with package name') try: if maint and not watch: maintenance.enable() packages = self.args if packages: if len(packages) > 1: watch = False for package in packages: _api.build(package, debug, mode, watch) else: _api.build_all(debug, mode) except (RuntimeError, _error.PackageNotRegistered, _error.PackageAlreadyRegistered) as e: raise console.error.CommandExecutionError(e) except KeyboardInterrupt: console.print_info('') finally: if maint and not watch: maintenance.disable()
def plugin_update(v_from: _Version): from pytsite import console if v_from < '1.4': from pytsite import mongodb for collection_name in mongodb.get_collection_names(): collection = mongodb.get_collection(collection_name) for doc in collection.find(): if '_ref' in doc: continue doc['_ref'] = '{}:{}'.format(doc['_model'], doc['_id']) collection.replace_one({'_id': doc['_id']}, doc) console.print_info('Document {} updated'.format(doc['_ref'])) if v_from < '1.6': console.run_command('odm:reindex') if v_from < '4.0': # Update all entities that have `Ref` and `RefsList` fields for m in get_registered_models(): console.print_info("Processing model '{}'".format(m)) fields_to_update = [] for f in dispense(m).fields.values(): if f.name != '_parent' and isinstance( f, (field.Ref, field.RefsList)): fields_to_update.append(f.name) if fields_to_update: for e in find(m).get(): for f_name in fields_to_update: e.f_set(f_name, e.f_get(f_name)) e.save(update_timestamp=False) console.print_info('Entity {} updated, fields {}'.format( e, fields_to_update)) if v_from < '6.0': console.run_command('odm:reindex')
def db_restore(): for model in _api.get_registered_models(): _api.clear_cache(model) console.print_info(lang.t('odm@cache_cleared', {'model': model}))
def exec(self): """Execute teh command """ model = self.arg(0) # Checking if the content model registered if not _api.is_model_registered(model): raise console.error.CommandExecutionError("'{}' is not a registered content model".format(model)) author_login = self.opt('author') num = self.opt('num') images_num = self.opt('images') language = self.opt('lang') no_html = self.opt('no-html') short = self.opt('short') title_len = self.opt('title-len') description_len = self.opt('description-len') if no_html: self.lorem_txt_args['format'] = 'text' if short: self.lorem_txt_args['paras'] = 1 users = list(auth.find_users(query.Query(query.Eq('status', 'active')), limit=10)) # Generate content entities for m in range(0, num): entity = _api.dispense(model) # Author if entity.has_field('author'): if author_login: author = auth.get_user(author_login) if not author: raise console.error.CommandExecutionError("'{}' is not a registered user".format(author_login)) else: if not users: raise console.error.CommandExecutionError(lang.t('content@no_users_found')) rand = randint(0, len(users) - 1) author = users[rand:rand + 1][0] entity.f_set('author', author.uid) # Title if entity.has_field('title'): entity.f_set('title', self._generate_title(title_len)) # Description if entity.has_field('description'): entity.f_set('description', self._generate_title(description_len)) # Body if entity.has_field('body'): body = [] for n in range(1, (images_num or 1) + 1): body.append(requests.get(self.lorem_txt_url, self.lorem_txt_args).content.decode('utf-8')) if not no_html and n > 1: body.append('\n<p>[img:{}]</p>\n'.format(n)) entity.f_set('body', ''.join(body)) # Images if entity.has_field('images') and images_num: for n in range(0, images_num): entity.f_add('images', file.create(self.lorem_img_url.format(randint(0, 1000)))) # Language if entity.has_field('language'): entity.f_set('language', language) # Status if entity.has_field('status'): entity.f_set('status', CONTENT_STATUS_PUBLISHED) events.fire('content@generate', entity=entity) entity.save() console.print_info(lang.t('content@new_content_created', {'model': entity.model, 'title': entity.title}))
def build(pkg_name: str, debug: bool = _DEBUG, mode: str = None, watch: bool = False): """Compile assets """ pkg_name = resolve_package(pkg_name) src = assets_src(pkg_name) dst = assets_dst(pkg_name) public_path = assets_public_path(pkg_name) timestamps_path = path.join(assets_dst('assetman'), 'timestamps.json') mode = mode or ('development' if debug else 'production') # Build translations if lang.is_package_registered(pkg_name): build_translations(pkg_name) # Building is possible only if 'webpack.config.js' exists webpack_config = path.join(src, 'webpack.config.js') if not path.exists(webpack_config): return # Clear destination directory if path.exists(dst): rmtree(dst) # Create output directory for timestamps file timestamps_dir_path = path.dirname(timestamps_path) if not path.exists(timestamps_dir_path): makedirs(timestamps_dir_path, 0o755, True) # It is important timestamps file to be exists before start building process if not path.isfile(timestamps_path): with open(timestamps_path, 'wt', encoding='utf-8') as f: f.write(json.dumps({})) # Collect webpack's config parts from all the packages webpack_parts = [] root_dir = reg.get('paths.root') + '/' for p in _packages.values(): if path.exists(path.join(p[0], 'webpack.part.js')): webpack_parts.append(p[0].replace(root_dir, '')) # Run webpack console.print_info( lang.t('assetman@compiling_assets_for_package', {'package': pkg_name})) args = [ '--mode', mode, '--config', webpack_config, '--context', assets_src(pkg_name), '--output-path', dst, '--output-public-path', public_path, '--env.NODE_ENV', mode, '--env.root_dir', root_dir, '--env.config_parts', ','.join(webpack_parts), '--watch', str(watch).lower(), ] _run_node_bin('webpack-cli', args, watch or debug) # Update timestamps _BUILD_TS.put(pkg_name, int(time.time())) with open(timestamps_path, 'wt', encoding='utf-8') as f: logger.debug(f"Writing timestamps into '{timestamps_path}'") f.write(json.dumps({k: _BUILD_TS.get(k) for k in _BUILD_TS.keys()}))
def on_pytsite_load(): update_info = _api.get_update_info() if not update_info: return # If there waiting updates exist, reload the application if _reg.get('env.type') == 'wsgi': _logger.warn('Application needs to be loaded in console to finish plugins update') return failed_plugins = [] # Call 'plugin_pre_install()' hooks for p_name, info in update_info.items(): v_to = _semver.Version(info['version_to']) try: # Check if the plugin is installed and loaded plugin = _api.get(p_name) # Call plugin_pre_install() hook if hasattr(plugin, 'plugin_pre_install') and callable(plugin.plugin_pre_install): plugin.plugin_pre_install() # Fire 'pre_install' event _events.fire('pytsite.plugman@pre_install', name=p_name, version=v_to) except _error.PluginNotLoaded as e: _logger.error(e) _console.print_warning(_lang.t('pytsite.plugman@plugin_install_error', { 'plugin': p_name, 'version': v_to, 'msg': str(e), })) failed_plugins.append(p_name) continue # Finish installing/updating plugins for p_name, info in update_info.items(): if p_name in failed_plugins: continue plugin = _api.get(p_name) v_from = _semver.Version(info['version_from']) v_to = _semver.Version(info['version_to']) try: _logger.info(_lang.t('pytsite.plugman@installing_plugin', { 'plugin': p_name, 'version': v_to, })) # Call plugin_install() hook if hasattr(plugin, 'plugin_install') and callable(plugin.plugin_install): plugin.plugin_install() # Fire 'install' event _events.fire('pytsite.plugman@install', name=p_name, version=v_to) _console.print_success(_lang.t('pytsite.plugman@plugin_install_success', { 'plugin': p_name, 'version': v_to, })) except Exception as e: _logger.error(e) _console.print_warning(_lang.t('pytsite.plugman@plugin_install_error', { 'plugin': p_name, 'version': v_to, 'msg': str(e), })) continue # Update plugin if v_from != '0.0.0': try: _console.print_info(_lang.t('pytsite.plugman@updating_plugin', { 'plugin': p_name, 'v_from': v_from, 'v_to': v_to, })) # Call plugin_update() hook if hasattr(plugin, 'plugin_update') and callable(plugin.plugin_update): plugin.plugin_update(v_from=v_from) # Fire 'update' event _events.fire('pytsite.plugman@update', name=p_name, v_from=v_from) _console.print_success(_lang.t('pytsite.plugman@plugin_update_success', { 'plugin': p_name, 'version': v_to, })) except Exception as e: _console.print_warning(_lang.t('pytsite.plugman@plugin_update_error', { 'plugin': p_name, 'version': v_to, 'msg': str(e), })) continue # Remove info from update queue _api.rm_update_info(p_name)
def on_pytsite_update_stage_2(): # Update all installed plugins _console.print_info(_lang.t('pytsite.plugman@updating_plugins')) _console.run_command('plugman:install', {'reload': False})
def exec(self, args: tuple = (), **kwargs): """Execute the command. """ app_path = _reg.get('paths.app') config_path = _reg.get('paths.config') stage = self.opt('stage') stop_after = self.opt('stop-after') _chdir(app_path) _maintenance.enable() d = self._get_update_data() if not d['update_in_progress']: d['pytsite_version_from'] = str(_package_info.version('pytsite')) d['app_version_from'] = str(_package_info.version('app')) d['update_in_progress'] = True self._set_update_data(d) # Stage 1: update pip and PytSite if stage == 1: _console.print_info(_lang.t('pytsite.update@updating_environment')) # Update pip _pip.install('pip', None, True, self.opt('debug')) # Update PytSite _pip.install('pytsite', _package_info.requires_pytsite('app'), True, self.opt('debug')) d['pytsite_version_to'] = str(_package_info.version('pytsite', False)) self._set_update_data(d) # Notify listeners _events.fire('pytsite.update@stage_1') if stop_after != 1: _subprocess.call(['./console', 'update', '--stage=2']) else: _maintenance.disable() # Stage 2: update application and configuration elif stage == 2: # Notify listeners about PytSite update _events.fire('pytsite.update@pytsite', v_from=_semver.Version(d['pytsite_version_from'])) # Update configuration if _path.exists(_path.join(config_path, '.git')): _console.print_info(_lang.t('pytsite.update@updating_configuration')) _subprocess_run(['git', '-C', config_path, 'pull']) # Update application if _path.exists(_path.join(app_path, '.git')): _console.print_info(_lang.t('pytsite.update@updating_application')) _subprocess_run(['git', '-C', app_path, 'pull']) d['app_version_to'] = str(_package_info.version('app', False)) self._set_update_data(d) # Notify listeners _events.fire('pytsite.update@stage_2') if stop_after != 2: _subprocess.call(['./console', 'update', '--stage=3']) else: _maintenance.disable() # Stage 3: finish update process elif stage == 3: _console.print_info(_lang.t('pytsite.update@applying_updates')) # Notify listeners about application update _events.fire('pytsite.update@app', v_from=_semver.Version(d['app_version_from'])) # Notify listeners _events.fire('pytsite.update@update') # Application's update hook import app if hasattr(app, 'app_update') and callable(app.app_update): app.app_update(v_from=_semver.Version(d['app_version_from'])) # Mark that update was finished successfully d['update_in_progress'] = False self._set_update_data(d) # Disable maintenance mode _maintenance.disable() # Reload the application _reload.reload()