def _update_pytsite(v_from: _semver.Version):
if v_from <= '7.9':
from os import path
from shutil import rmtree
# New dict key added to all file storage items, so entire file cache must be cleared
rmtree(_reg.get('cache.file_driver_storage', path.join(_reg.get('paths.storage'), 'cache')))
def _dump(self):
if _subprocess.call('which mongodump', stdout=_subprocess.DEVNULL, stderr=_subprocess.DEVNULL, shell=True) != 0:
raise RuntimeError('Cannot find mongodump executable.')
_maintenance.enable()
db_name = _reg.get('db.database')
target_dir = _path.join(_reg.get('paths.root'), 'misc', 'dbdump')
target_subdir = _path.join(target_dir, db_name)
if _path.exists(target_subdir):
ctime = _datetime.fromtimestamp(_path.getctime(target_subdir))
target_subdir_move = '{}-{}'.format(target_subdir, ctime.strftime('%Y%m%d-%H%M%S'))
_shutil.move(target_subdir, target_subdir_move)
from . import _api
config = _api.get_config()
command = 'mongodump -h {}:{} --gzip -o {} -d {}'.format(config['host'], config['port'], target_dir, db_name)
if config['user']:
command += ' -u {} -p {}'.format(config['user'], config['password'])
if config['ssl']:
command += ' --ssl --sslAllowInvalidCertificates'
r = _subprocess.call(command, shell=True)
_maintenance.disable()
return r
def _restore(self):
if _subprocess.call('which mongorestore', stdout=_subprocess.DEVNULL, stderr=_subprocess.DEVNULL, shell=True):
raise RuntimeError('Cannot find mongorestore executable.')
_maintenance.enable()
db_name = _reg.get('db.database')
source_dir = _path.join(_reg.get('paths.root'), 'misc', 'dbdump', db_name)
from . import _api
config = _api.get_config()
command = 'mongorestore -h {}:{} --drop --gzip --stopOnError --dir {} -d {}'. \
format(config['host'], config['port'], source_dir, db_name)
if config['user']:
command += ' -u {} -p {}'.format(config['user'], config['password'])
if config['ssl']:
command += ' --ssl --sslAllowInvalidCertificates'
r = _subprocess.call(command, shell=True)
_events.fire('pytsite.mongodb@restore')
_maintenance.disable()
return r
def router_dispatch():
"""'pytsite.router.dispatch' event handler.
"""
if _auth.get_current_user().has_role('dev'):
if not _reg.get('tumblr.app_key') or not _reg.get('tumblr.app_secret'):
_router.session().add_warning_message(
_lang.t('tumblr@plugin_setup_required_warning'))
def exec(self) -> str:
theme_v = reg.get('theme.version', '1')
assetman.preload('v1/css/content-entity-modify.css'.format(theme_v))
return tpl.render(
'v1/content-entity-modify'.format(reg.get('theme.version', '1')),
self.args)
def exec(self):
verbosity = 2 if _reg.get('debug') else 1
if not self.args:
raise _console.error.MissingArgument('pytsite.testing@target_is_not_specified')
for target in self.args:
argv = [_sys.argv[0]]
if self.opt('no-discover') or '.tests.' in target:
argv.append(target)
_unittest.TestProgram(None, argv=argv, failfast=True, verbosity=verbosity)
else:
argv += ['discover', '-s']
if target.startswith('pytsite.'):
target = _path.join(_reg.get('paths.pytsite'), target.replace('pytsite.', ''), 'tests')
else:
target = _path.join(_reg.get('paths.root'), target.replace('.', _path.sep), 'tests')
if not _path.exists(target):
raise _console.error.CommandExecutionError('Directory {} is not found'.format(target))
argv.append(target)
_unittest.TestProgram(None, argv=argv, failfast=True, verbosity=verbosity)
def get_comments_count(self, thread_uid: str) -> int:
"""Get comments count for particular thread.
"""
short_name = reg.get('disqus.short_name')
secret_key = reg.get('disqus.secret_key')
if not short_name or not secret_key:
return 0
count = 0
try:
r = requests.get(
'https://disqus.com/api/3.0/forums/listThreads.json', {
'api_secret': secret_key,
'forum': short_name,
'thread': 'ident:' + thread_uid,
'limit': 1,
}).json()
if r['code'] == 0 and r['response']:
count = r['response'][0]['posts']
except Exception as e:
logger.error(e)
return count
def cleanup():
success, failed = _util.cleanup_files(_reg.get('paths.log'), _reg.get('logger.file_ttl', 2592000)) # 30d
for f_path in success:
_logger.debug('Obsolete log file removed: {}'.format(f_path))
for f_path, e in failed:
_logger.error('Error while removing obsolete log file {}: {}'.format(f_path, e))
def on_cleanup():
success, failed = _util.cleanup_files(_reg.get('paths.session'), _reg.get('router.session_ttl', 86400))
for f_path in success:
_logger.debug('Obsolete session file removed: {}'.format(f_path))
for f_path, e in failed:
_logger.error('Error while removing obsolete session file {}: {}'.format(f_path, e))
def get_app_secret() -> str:
"""Get application's secret key.
"""
app_secret = _reg.get('facebook.app_secret') or _reg.get('facebook.app_secret')
if not app_secret:
raise _error.AppSecretNotSet("Configuration parameter 'facebook.app_secret' is not set")
return app_secret
def get_app_id() -> str:
"""Get application's ID.
"""
app_id = _reg.get('facebook.app_id') or _reg.get('facebook.app_id')
if not app_id:
raise _error.AppIdNotSet("Configuration parameter 'facebook.app_id' is not set")
return app_id
def _generate_feeds():
# For each language we have separate feed
for lng in lang.langs():
# Generate RSS feed for each model
for model in reg.get('content.rss_models', ()):
filename = 'rss-{}'.format(model)
_api.generate_rss(model,
filename,
lng,
length=reg.get('content.feed_length', 20))
def get_app_key() -> str:
"""Get application's key.
"""
app_key = _reg.get('tumblr.app_key') or _reg.get('tumblr.app_key')
if not app_key:
raise _error.AppKeyNotSet(
"Configuration parameter 'tumblr.app_key' is not set")
return app_key
def get_app_id() -> str:
"""Get application's key.
"""
app_id = _reg.get('vkontakte.app_id') or _reg.get('vkontakte.app_id')
if not app_id:
raise _error.AppIdNotSet(
"Configuration parameter 'vkontakte.app_id' is not set")
return app_id
def pytsite_cleanup():
root = _path.join(_reg.get('paths.static'), 'image', 'resize')
ttl = _reg.get('file_storage_odm.static_ttl', 2592000) # 1 month
success, failed = _util.cleanup_files(root, ttl)
for f_path in success:
_logger.debug('Obsolete static file removed: {}'.format(f_path))
for f_path, e in failed:
_logger.error('Error while removing obsolete static file {}: {}'.format(f_path, e))
def __init__(self):
"""Init
"""
from pytsite import router as _router
self._server_name = _router.server_name()
self._path = _reg.get('cache.file_driver_storage', _path.join(_reg.get('paths.storage'), 'cache'))
# Create cache directory
if not _path.exists(self._path):
_makedirs(self._path, 0o755, True)
def __init__(self, access_token: str, app_id: str = None, app_secret: str = None):
"""Init.
"""
self._access_token = access_token
self._app_id = app_id or _reg.get('vkontakte.app_id')
if not self._app_id:
raise RuntimeError('Application ID is not defined.')
self._app_secret = app_secret or _reg.get('vkontakte.app_secret')
if not self._app_id:
raise RuntimeError('Application secret key is not defined.')
def plugin_load():
from pytsite import reg, util
from plugins import auth, odm
from . import _driver
# ODM models
role_cls = reg.get('auth_storage_odm.role_odm_class', 'plugins.auth_storage_odm.ODMRole')
user_cls = reg.get('auth_storage_odm.user_odm_class', 'plugins.auth_storage_odm.ODMUser')
odm.register_model('role', util.get_module_attr(role_cls))
odm.register_model('user', util.get_module_attr(user_cls))
odm.register_model('follower', ODMFollower)
odm.register_model('blocked_user', ODMBlockedUser)
# Register storage driver
auth.register_storage_driver(_driver.Storage())
def content_on_status_change(self):
"""Hook
"""
# Update publish time if entity is being published
now = datetime.now()
if self.prev_status == CONTENT_STATUS_UNPUBLISHED and \
self.status in (CONTENT_STATUS_WAITING, CONTENT_STATUS_PUBLISHED) and \
self.publish_time < now:
self.f_set('publish_time', now).save(fast=True)
if reg.get('content.waiting_status_admin_notification', True):
self._content_notify_admins_waiting_status()
if reg.get('content.status_change_author_notification', True):
self._content_notify_author_status_change()
def _init():
from os import mkdir
from pytsite import reg, lang, tpl, console, update, on_pytsite_load
from . import _cc, _eh
# Resources
lang.register_package(__name__)
tpl.register_global('plugins', _PluginsTplGlobal())
# Create 'plugins' package if it doesn't exist
plugins_dir = plugins_dir_path()
if not _path.exists(plugins_dir):
mkdir(plugins_dir, 0o755)
with open(_path.join(plugins_dir, '__init__.py'), 'wt') as f:
f.write('"""Pytsite Plugins\n"""\n')
# Register console commands
console.register_command(_cc.Install())
console.register_command(_cc.Uninstall())
# Enable imports checking
_meta_path.insert(0, _MetaPathHook())
# Load installed plugins
if reg.get('plugman.autoload', True):
maint_was_enabled = _maintenance.is_enabled()
disabled_plugins = reg.get('plugman.disabled_plugins', [])
try:
_maintenance.enable(True)
for p_name in local_plugins_info():
if p_name.startswith('_') or p_name in disabled_plugins:
continue
try:
if not is_loaded(p_name):
load(p_name)
except (error.PluginLoadError, error.PluginNotInstalled) as e:
console.print_warning(e)
finally:
if not maint_was_enabled:
_maintenance.disable(True)
# Event handlers
on_pytsite_load(_eh.on_pytsite_load)
update.on_update_stage_2(_eh.on_pytsite_update_stage_2)
def exec(self) -> str:
theme_v = reg.get('theme.version', '1')
e = self.arg('entity')
exclude_ids = [e.id]
self.args.update({
'entity_tags':
tag.widget.EntityTagCloud('entity-tag-cloud',
entity=e,
term_css=''),
'related_1':
_get_articles(exclude_ids, 3, e.section, 'views_count')
if e.model == 'article' else [],
'related_2':
_get_articles(exclude_ids, 2, e.section, 'views_count')
if e.model == 'article' else [],
'related_3':
_get_articles(exclude_ids, 2, e.section)
if e.model == 'article' else [],
})
if e.images:
self.args.update({
'page_header_article':
e,
'page_header_article_tags':
tag.widget.EntityTagCloud('header-article-tags',
entity=e,
term_css=''),
'fullscreen_page_header':
True,
})
if plugman.is_installed('addthis'):
from plugins import addthis
self.args.update({
'share_widget':
addthis.widget.AddThis('add-this-share')
if reg.get('addthis.pub_id') else '',
})
if plugman.is_installed('disqus'):
self.args.update(
{'comments_widget': comments.get_widget(driver_name='disqus')})
assetman.preload('v{}/css/content-entity-view.css'.format(theme_v))
return tpl.render('v{}/content-entity-view'.format(theme_v), self.args)
def _init():
from os import path, makedirs
from pytsite import tpl, lang, reg, cleanup
from . import _eh
# Resources
tpl.register_package(__name__)
lang.register_package(__name__)
# Create directory to store session data
session_storage_path = reg.get('paths.session')
if not path.exists(session_storage_path):
makedirs(session_storage_path, 0o755, True)
# Lang globals
lang.register_global('base_url', lambda language, args: base_url(language))
# Tpl globals
tpl.register_global('url', url)
tpl.register_global('is_rule_defined', has_rule)
tpl.register_global('rule_url', rule_url)
tpl.register_global('current_url', current_url)
tpl.register_global('current_path', current_path)
tpl.register_global('base_url', base_url)
tpl.register_global('is_base_url', is_base_url)
tpl.register_global('is_main_host', is_main_host)
tpl.register_global('session_messages', lambda x: session().get_messages(x) if session() else ())
# Clear flash messages from all sessions
s_store = get_session_store()
for sid in s_store.list():
s_store.save_if_modified(s_store.get(sid).clear_messages())
# Events handlers
cleanup.on_cleanup(_eh.on_cleanup)
def exec(self) -> str:
theme_v = reg.get('theme.version', '1')
self.args.update(content.paginate(self.arg('finder')))
assetman.preload('v{}/css/content-entity-index.css'.format(theme_v))
return tpl.render('v{}/content-entity-index'.format(theme_v),
self.args)
def register_package(package_name: str, assets_dir: str = 'res/assets'):
"""Register PytSite package which contains assets
"""
pkg_spec = find_spec(package_name)
if not pkg_spec:
raise RuntimeError("Package '{}' is not found".format(package_name))
# Check whether assetman's package is already registered
if package_name in _packages:
raise _error.PackageAlreadyRegistered(package_name)
# Absolute path to package's assets source directory
src_path = path.abspath(
path.join(path.dirname(pkg_spec.origin), assets_dir))
if not path.isdir(src_path):
FileNotFoundError("Directory '{}' is not found".format(src_path))
# Absolute path to package's assets destination directory
assets_path = reg.get('paths.assets')
if not assets_path:
raise RuntimeError(
"It seems you call register_package('{}') too early".format(
package_name))
dst_path = path.join(assets_path, package_name)
_packages[package_name] = (src_path, dst_path)
def __init__(self, uid: str, **kwargs):
"""Init
"""
super().__init__(uid, **kwargs)
self._form_group = False
self._has_messages = False
self._pub_id = reg.get('addthis.pub_id')
if not self._pub_id:
raise RuntimeError("Setting 'addthis.pub_id' is not defined")
self._box_type = kwargs.get('box_type', _valid_box_types[0])
if self._box_type not in _valid_box_types:
raise RuntimeError(
"Invalid type: '{}'. Valid types are: {}.".format(
self._box_type, str(_valid_box_types)))
if not self._box_type.startswith('addthis_'):
self._box_type = 'addthis_' + self._box_type
# To support previous version of AddThis container naming convention
if self._box_type == 'addthis_inline_share_toolbox':
self._box_type += ' addthis_sharing_toolbox'
self._url = kwargs.get('url')
self._css += ' widget-addthis'
def register_model(model: str,
cls,
menu_title: str = None,
menu_weight: int = 0,
menu_icon: str = 'fa fas fa-tags',
menu_sid: str = 'taxonomy',
menu_roles: Union[str, list, tuple] = ('admin', 'dev'),
menu_permissions: Union[str, list, tuple] = None):
"""Register a taxonomy model
"""
if model in _models:
raise RuntimeError(
"Taxonomy model '{}' is already registered".format(model))
if isinstance(cls, str):
cls = util.get_module_attr(cls)
if not issubclass(cls, Term):
raise TypeError('Subclass of {} expected'.format(Term))
odm.register_model(model, cls)
_models.append(model)
if reg.get('env.type') == 'wsgi' and menu_title:
menu_url = router.rule_path('odm_ui@admin_browse', {'model': model})
admin.sidebar.add_menu(
menu_sid,
model,
menu_title,
menu_url,
menu_icon,
weight=menu_weight,
roles=menu_roles,
permissions=menu_permissions,
)
def _build_store_path(name: str, mime: str, propose: str = None) -> str:
"""Build unique path to store file on the filesystem.
"""
storage_dir = _os.path.join(_reg.get('paths.storage'), 'file', mime.split('/')[0])
ext = _os.path.splitext(name)[1]
rnd_str = _util.random_str
# Possible (but not final) path
possible_target_path = _os.path.join(storage_dir, rnd_str(2), rnd_str(2), rnd_str()) + ext
# Check if the proposed path suits the requirements
if propose:
m = _re.match('(\w{2})/(\w{2})/(\w{16})(\.\w+)$', propose)
if m:
ext = m.group(4)
possible_target_path = _os.path.join(storage_dir, m.group(1), m.group(2), m.group(3)) + ext
# Search for path which doesn't exist on the filesystem
while True:
if not _os.path.exists(possible_target_path):
store_path = possible_target_path
break
else:
possible_target_path = _os.path.join(storage_dir, rnd_str(2), rnd_str(2), rnd_str()) + ext
return store_path
def get_client_id() -> str:
client_id = reg.get('auth_id_gov_ua.client_id')
if not client_id:
raise RuntimeError(
'auth_id_gov_ua.client_id configuration parameter is not defined')
return client_id
def __init__(self):
self._role_cls = util.get_module_attr(
reg.get(_REG_ROLE_CLS, 'plugins.auth_storage_odm.Role'))
if not issubclass(self._role_cls, _model.Role):
raise TypeError(
"Subclass of {} expected, got {}. Please check the '{}' configuration parameter"
.format(auth.AbstractRole, type(self._role_cls),
_REG_ROLE_CLS))
self._user_cls = util.get_module_attr(
reg.get(_REG_USER_CLS, 'plugins.auth_storage_odm.User'))
if not issubclass(self._user_cls, _model.User):
raise TypeError(
"Subclass of {} expected, got {}. Please check the '{}' configuration parameter"
.format(auth.AbstractUser, type(self._user_cls),
_REG_USER_CLS))
def register_model(model: str, cls: Union[str, ContentModelClass], title: str, menu_weight: int = 0,
menu_icon: str = 'fa fa-file-text-o', menu_sid: str = 'content', replace: bool = False):
"""Register content model
"""
# Resolve class
if isinstance(cls, str):
cls = util.get_module_attr(cls) # type: ContentModelClass
if not issubclass(cls, Content):
raise TypeError('Subclass of {} expected, got {}'.format(Content, type(cls)))
if not replace and is_model_registered(model):
raise KeyError("Content model '{}' is already registered".format(model))
# Register ODM model
odm.register_model(model, cls, replace)
# Saving info about registered _content_ model
_models[model] = (cls, title)
if reg.get('env.type') == 'wsgi':
mock = dispense(model)
perms = ['odm_auth@{}.{}'.format(p, model) for p in mock.odm_auth_permissions()],
admin.sidebar.add_menu(
sid=menu_sid,
mid=model,
title=title,
path=router.rule_path('odm_ui@admin_browse', {'model': model}),
icon=menu_icon,
weight=menu_weight,
permissions=perms,
replace=replace,
)
def logo_url(width: int = 0, height: int = 0, enlarge: bool = False):
s = reg.get('theme.logo')
try:
return file.get(s).get_url(width=width, height=height, enlarge=enlarge) if s else \
assetman.url('$theme@img/appicon.png')
except file.error.FileNotFound:
return assetman.url('$theme@img/appicon.png')
def install_npm_deps(package_names: Union[str, List[str]]):
"""Install NPM packages required by locally installed plugins
"""
is_dev_host = path.isdir(path.join(reg.get('paths.root'), 'npm_packages'))
if isinstance(package_names, str):
package_names = [package_names]
# Build list of NPM packages required by plugins
npm_pkgs_to_install = []
for pkg_name in package_names:
# Skip package if it does not provide package.json
json_path = path.join(assets_src(pkg_name), 'package.json')
if not path.exists(json_path):
continue
# Collect dependencies
json = util.load_json(json_path)
for name, ver in json.get('dependencies', {}).items():
if name.startswith('@pytsite') and is_dev_host:
continue
npm_pkg_spec = '{}@{}'.format(name, ver)
if npm_pkg_spec not in npm_pkgs_to_install:
npm_pkgs_to_install.append(npm_pkg_spec)
# Install required NPM packages
npm_install(npm_pkgs_to_install)
def plugin_load():
from pytsite import reg
from plugins import menu
from . import _model
if reg.get('content_menu.register_default_model', True):
menu.register_model('content_menu', _model.ContentMenu, __name__ + '@content')
def on_tpl_render(tpl_name: str, args: dict):
args.update({
'theme_v':
reg.get('theme.version', '1'),
'top_navbar_items_num':
int(reg.get('theme.top_navbar_items_num', '5')),
'language_nav':
widget.select.LanguageNav('language-nav'),
})
if plugman.is_installed(['page', 'section']):
from plugins import content, section
args['content_sections'] = list(section.get())
if content.is_model_registered('page'):
args['content_pages'] = list(content.find('page').get())
def update_stage_2():
# Install/update pip packages
_console.print_info(_lang.t('pytsite.pip@updating_packages'))
for pkg_name, pkg_ver in _package_info.requires_packages('app').items():
try:
_api.install(pkg_name, pkg_ver, True, _reg.get('debug'))
except _error.PackageInstallError as e:
raise _console.error.CommandExecutionError(e)
def _cleanup_tmp_files():
success, failed = _util.cleanup_files(_reg.get('paths.tmp'), 86400) # 24h
for f_path in success:
_logger.debug('Obsolete tmp file removed: {}'.format(f_path))
for f_path, e in failed:
_logger.error('Error while removing obsolete tmp file {}: {}'.format(f_path, e))
def exec(self):
del self.args['_pytsite_router_rule_name']
endpoint = '/' + self.args.pop('http_api_endpoint')
current_path = router.current_path(False)
request_method = router.request().method
# Switch language
language = router.request().headers.get('Accept-Language') # type: str
if language:
for lng in language.split(','):
lng = lng.strip()
if not lng.startswith('q=') and lang.is_defined(language):
lang.set_current(language)
break
try:
events.fire('http_api@pre_request')
rule = _api.match(router.request().method, endpoint)
events.fire('http_api@request')
controller = rule.controller_class() # type: routing.Controller
controller.request = self.request
controller.args.update(self.args)
controller.args.update(rule.args)
controller.args['_pytsite_http_api_rule_name'] = rule.name
controller.args.validate()
response = controller.exec()
return response if isinstance(
response, http.Response) else http.JSONResponse(response)
except (http.error.Base, errors.ForbidOperation) as e:
if reg.get('debug'):
logger.error(e)
else:
logger.error('{} {}: {}'.format(request_method, current_path,
e.description))
if isinstance(e, errors.ForbidOperation):
e = http.error.Forbidden(e)
if e.response and isinstance(e.response, http.JSONResponse):
response = e.response
response.status_code = e.code
else:
# It is important to do `str(e.description)`, because `e.description` might be an exception
response = http.JSONResponse({'error': str(e.description)},
e.code)
return response
except UserWarning as e:
logger.warn('{} {}: {}'.format(request_method, current_path, e))
return http.JSONResponse({'warning': str(e)}, 200)
except Exception as e:
logger.error('{} {}: {}'.format(request_method, current_path, e),
exc_info=e)
return http.JSONResponse({'error': str(e)}, 500)
def __init__(self, uid: str, **kwargs):
"""Init.
"""
super().__init__(uid, **kwargs)
self._short_name = reg.get('disqus.short_name')
self._thread_uid = kwargs.get('thread_uid')
self._css += ' widget-disqus'
def plugin_load():
from pytsite import reg
from plugins import admin
from . import _api
admin.sidebar.add_section('menu', __name__ + '@menu')
if reg.get('menu.register_default_model', True):
_api.register_model('menu', _model.Menu, __name__ + '@menu')
def router_dispatch():
"""'pytsite.router.dispatch' handler.
"""
t_id = _reg.get('google_analytics.tracking_id')
if not t_id and _auth.get_current_user().has_role('dev'):
_router.session().add_warning_message(
_lang.t('google_analytics@plugin_setup_required_warning'))
else:
_assetman.inline_js(
_tpl.render('google_analytics@counter', {'tracking_id': t_id}))
def plugin_load_wsgi():
from pytsite import router, reg
from plugins import settings
from . import _settings_form
settings.define('mail', _settings_form.Form, 'mail_settings@mail',
'fa fa-envelope', 'dev')
if not reg.get('mail.from'):
reg.put('mail.from', 'info@' + router.server_name())
def _do_reload():
"""Modify 'touch.reload' file
"""
touch_reload_path = _path.join(_reg.get('paths.storage'), 'touch.reload')
_events.fire('pytsite.reload@before_reload')
with open(touch_reload_path, 'wt') as f:
f.write(_util.w3c_datetime_str())
_events.fire('pytsite.reload@reload')
_console.print_info(_lang.t('pytsite.reload@app_is_reloading'))
def cron_every_min():
out = ''
for r in _events.fire('pytsite.stats@update'):
if not r or not isinstance(r, str):
continue
out += '- ' + r + '\n'
with open(_path.join(_reg.get('paths.storage'), 'stats.txt'), 'wt') as f:
f.write(_util.w3c_datetime_str() + '\n\n' + out)
if out:
_logger.info('Current stats:\n{}'.format(out[:-1]))
def mk_tmp_dir(suffix: str = None, prefix: str = None, subdir: str = None) -> str:
from pytsite import reg
tmp_root = reg.get('paths.tmp')
if not tmp_root:
raise RuntimeError('Cannot determine temporary directory location')
if subdir:
tmp_root = _path.join(tmp_root, subdir)
if not _path.exists(tmp_root):
_makedirs(tmp_root, 0o755)
return _mkdtemp(suffix, prefix, tmp_root)
def mk_tmp_file(suffix: str = None, prefix: str = None, subdir: str = None, text: bool = False) -> _Tuple[int, str]:
"""Create temporary file
Returns tuple of two items: file's descriptor and absolute path.
"""
from pytsite import reg
tmp_dir = reg.get('paths.tmp')
if not tmp_dir:
raise RuntimeError('Cannot determine temporary directory location')
if subdir:
tmp_dir = _path.join(tmp_dir, subdir)
if not _path.exists(tmp_dir):
_makedirs(tmp_dir, 0o755)
return _mkstemp(suffix, prefix, tmp_dir, text)
def get_config() -> dict:
"""
:return:
"""
return {
'host': _reg.get('db.host', 'localhost'),
'port': _reg.get('db.port', 27017),
'ssl': _reg.get('db.ssl', False),
'database': _reg.get('db.database', 'test'),
'user': _reg.get('db.user'),
'password': _reg.get('db.password'),
'connect_timeout': 5000,
'socket_timeout': 5000,
'server_selection_timeout': 5000,
}
"""PytSite Logger API Functions
"""
__author__ = 'Oleksandr Shepetko'
__email__ = '*****@*****.**'
__license__ = 'MIT'
import logging as _logging
from pytsite import reg as _reg
_logger = _logging.getLogger(_reg.get('env.name', 'default'))
def _log(level: int, msg, *args, **kwargs):
if isinstance(msg, Exception):
kwargs['exc_info'] = msg
_logger.log(level, msg, *args, **kwargs)
def debug(msg, *args, **kwargs):
_log(_logging.DEBUG, msg, *args, **kwargs)
def info(msg, *args, **kwargs):
_log(_logging.INFO, msg, *args, **kwargs)
def warn(msg, *args, **kwargs):
_log(_logging.WARNING, msg, *args, **kwargs)
def _init():
from pytsite import reg, cron
if reg.get('debug'):
from . import _eh
cron.every_min(_eh.cron_every_min)
"""PytSite Logger
"""
__author__ = 'Oleksandr Shepetko'
__email__ = '*****@*****.**'
__license__ = 'MIT'
import logging as _logging
from os import path as _path, makedirs as _makedirs
from datetime import datetime as _datetime
from pytsite import reg as _reg, cleanup as _cleanup
_log_dir = _reg.get('paths.log')
_log_path = _path.join(_log_dir, _datetime.now().strftime('{}-%Y%m%d.log'.format(_reg.get('env.type'))))
_log_level = _logging.DEBUG if _reg.get('debug') else _logging.INFO
if not _path.exists(_log_dir):
_makedirs(_log_dir, 0o755, True)
# Create logger
_logger = _logging.getLogger(_reg.get('env.name', 'default'))
_logger.setLevel(_log_level)
# Setup handler
_handler = _logging.FileHandler(_log_path, encoding='utf-8')
if _log_level == _logging.DEBUG:
fmt = '%(asctime)s %(levelname)7s %(process)d:%(thread)d %(message)s'
else:
fmt = '%(asctime)s %(levelname)7s %(message)s'
_handler.setFormatter(_logging.Formatter(fmt))
_handler.setLevel(_log_level)
"""PytSite Cache
"""
__author__ = 'Oleksandr Shepetko'
__email__ = '*****@*****.**'
__license__ = 'MIT'
# Public API
from ._api import set_driver, has_pool, create_pool, get_pool, get_pools, cleanup
from . import _driver as driver, _error as error
from ._pool import Pool
from pytsite import reg as _reg, threading as _threading, update as _update, semver as _semver
if _reg.get('env.type') == 'wsgi':
def _cleanup_worker():
cleanup()
_threading.run_in_thread(_cleanup_worker, 60)
_threading.run_in_thread(_cleanup_worker, 60)
def _update_pytsite(v_from: _semver.Version):
if v_from <= '7.9':
from os import path
from shutil import rmtree
# New dict key added to all file storage items, so entire file cache must be cleared
rmtree(_reg.get('cache.file_driver_storage', path.join(_reg.get('paths.storage'), 'cache')))
"""PytSite Cache API
"""
__author__ = 'Oleksandr Shepetko'
__email__ = '*****@*****.**'
__license__ = 'MIT'
from typing import Dict as _Dict
from pytsite import logger as _logger, reg as _reg
from . import _error
from ._driver import Abstract as _AbstractDriver
from ._pool import Pool as _Pool
_current_driver = None # type: _AbstractDriver
_pools = {} # type: _Dict[str, _Pool]
_dbg = _reg.get('cache.debug')
def set_driver(driver: _AbstractDriver):
"""Register a cache driver
"""
global _current_driver
if not isinstance(driver, _AbstractDriver):
raise TypeError('Instance of {} expected, got {}'.format(_AbstractDriver, type(driver)))
# When switching from one driver to another, it is important to move existing keys to the new storage
keys_to_move = {} # type: _Dict[str, list]
if _current_driver:
for pool in _pools.values():
keys_to_move[pool.uid] = []
for key in pool.keys():
def __init__(self, access_token: str = None):
self._access_token = access_token or _reg.get('github.access_token')
def _data_path(self) -> str:
return _path.join(_reg.get('paths.storage'), 'pytsite.update')
def exec(self, args: tuple = (), **kwargs):
"""Execute the command.
"""
app_path = _reg.get('paths.app')
config_path = _reg.get('paths.config')
stage = self.opt('stage')
stop_after = self.opt('stop-after')
_chdir(app_path)
_maintenance.enable()
d = self._get_update_data()
if not d['update_in_progress']:
d['pytsite_version_from'] = str(_package_info.version('pytsite'))
d['app_version_from'] = str(_package_info.version('app'))
d['update_in_progress'] = True
self._set_update_data(d)
# Stage 1: update pip and PytSite
if stage == 1:
_console.print_info(_lang.t('pytsite.update@updating_environment'))
# Update pip
_pip.install('pip', None, True, self.opt('debug'))
# Update PytSite
_pip.install('pytsite', _package_info.requires_pytsite('app'), True, self.opt('debug'))
d['pytsite_version_to'] = str(_package_info.version('pytsite', False))
self._set_update_data(d)
# Notify listeners
_events.fire('pytsite.update@stage_1')
if stop_after != 1:
_subprocess.call(['./console', 'update', '--stage=2'])
else:
_maintenance.disable()
# Stage 2: update application and configuration
elif stage == 2:
# Notify listeners about PytSite update
_events.fire('pytsite.update@pytsite', v_from=_semver.Version(d['pytsite_version_from']))
# Update configuration
if _path.exists(_path.join(config_path, '.git')):
_console.print_info(_lang.t('pytsite.update@updating_configuration'))
_subprocess_run(['git', '-C', config_path, 'pull'])
# Update application
if _path.exists(_path.join(app_path, '.git')):
_console.print_info(_lang.t('pytsite.update@updating_application'))
_subprocess_run(['git', '-C', app_path, 'pull'])
d['app_version_to'] = str(_package_info.version('app', False))
self._set_update_data(d)
# Notify listeners
_events.fire('pytsite.update@stage_2')
if stop_after != 2:
_subprocess.call(['./console', 'update', '--stage=3'])
else:
_maintenance.disable()
# Stage 3: finish update process
elif stage == 3:
_console.print_info(_lang.t('pytsite.update@applying_updates'))
# Notify listeners about application update
_events.fire('pytsite.update@app', v_from=_semver.Version(d['app_version_from']))
# Notify listeners
_events.fire('pytsite.update@update')
# Application's update hook
import app
if hasattr(app, 'app_update') and callable(app.app_update):
app.app_update(v_from=_semver.Version(d['app_version_from']))
# Mark that update was finished successfully
d['update_in_progress'] = False
self._set_update_data(d)
# Disable maintenance mode
_maintenance.disable()
# Reload the application
_reload.reload()
def on_pytsite_load():
update_info = _api.get_update_info()
if not update_info:
return
# If there waiting updates exist, reload the application
if _reg.get('env.type') == 'wsgi':
_logger.warn('Application needs to be loaded in console to finish plugins update')
return
failed_plugins = []
# Call 'plugin_pre_install()' hooks
for p_name, info in update_info.items():
v_to = _semver.Version(info['version_to'])
try:
# Check if the plugin is installed and loaded
plugin = _api.get(p_name)
# Call plugin_pre_install() hook
if hasattr(plugin, 'plugin_pre_install') and callable(plugin.plugin_pre_install):
plugin.plugin_pre_install()
# Fire 'pre_install' event
_events.fire('pytsite.plugman@pre_install', name=p_name, version=v_to)
except _error.PluginNotLoaded as e:
_logger.error(e)
_console.print_warning(_lang.t('pytsite.plugman@plugin_install_error', {
'plugin': p_name,
'version': v_to,
'msg': str(e),
}))
failed_plugins.append(p_name)
continue
# Finish installing/updating plugins
for p_name, info in update_info.items():
if p_name in failed_plugins:
continue
plugin = _api.get(p_name)
v_from = _semver.Version(info['version_from'])
v_to = _semver.Version(info['version_to'])
try:
_logger.info(_lang.t('pytsite.plugman@installing_plugin', {
'plugin': p_name,
'version': v_to,
}))
# Call plugin_install() hook
if hasattr(plugin, 'plugin_install') and callable(plugin.plugin_install):
plugin.plugin_install()
# Fire 'install' event
_events.fire('pytsite.plugman@install', name=p_name, version=v_to)
_console.print_success(_lang.t('pytsite.plugman@plugin_install_success', {
'plugin': p_name,
'version': v_to,
}))
except Exception as e:
_logger.error(e)
_console.print_warning(_lang.t('pytsite.plugman@plugin_install_error', {
'plugin': p_name,
'version': v_to,
'msg': str(e),
}))
continue
# Update plugin
if v_from != '0.0.0':
try:
_console.print_info(_lang.t('pytsite.plugman@updating_plugin', {
'plugin': p_name,
'v_from': v_from,
'v_to': v_to,
}))
# Call plugin_update() hook
if hasattr(plugin, 'plugin_update') and callable(plugin.plugin_update):
plugin.plugin_update(v_from=v_from)
# Fire 'update' event
_events.fire('pytsite.plugman@update', name=p_name, v_from=v_from)
_console.print_success(_lang.t('pytsite.plugman@plugin_update_success', {
'plugin': p_name,
'version': v_to,
}))
except Exception as e:
_console.print_warning(_lang.t('pytsite.plugman@plugin_update_error', {
'plugin': p_name,
'version': v_to,
'msg': str(e),
}))
continue
# Remove info from update queue
_api.rm_update_info(p_name)
"""PytSite pip Console Commands
"""
__author__ = 'Oleksandr Shepetko'
__email__ = '*****@*****.**'
__license__ = 'MIT'
import re as _re
from pytsite import reg as _reg, console as _console, pip as _pip, package_info as _package_info, lang as _lang
_PKG_SPEC_RE = _re.compile('^([a-zA-Z0-9\-_]+)([~=!<>]+[0-9]+(?:\.[0-9]+)*)?')
_DEBUG = _reg.get('debug')
class Install(_console.Command):
def __init__(self):
super().__init__()
self.define_option(_console.option.Bool('upgrade'))
self.define_option(_console.option.Bool('debug', default=_DEBUG))
@property
def name(self) -> str:
"""Get name of the command
"""
return 'pip:install'
@property
def description(self) -> str:
"""Get description of the command.
"""
return 'pytsite.pip@install_console_command_description'
def mail_from() -> tuple:
"""Get default mail sender's address and name.
"""
return _reg.get('mail.from', 'info@' + _router.server_name())
