def _restore(self): if _subprocess.call('which mongorestore', stdout=_subprocess.DEVNULL, stderr=_subprocess.DEVNULL, shell=True): raise RuntimeError('Cannot find mongorestore executable.') _maintenance.enable() db_name = _reg.get('db.database') source_dir = _path.join(_reg.get('paths.root'), 'misc', 'dbdump', db_name) from . import _api config = _api.get_config() command = 'mongorestore -h {}:{} --drop --gzip --stopOnError --dir {} -d {}'. \ format(config['host'], config['port'], source_dir, db_name) if config['user']: command += ' -u {} -p {}'.format(config['user'], config['password']) if config['ssl']: command += ' --ssl --sslAllowInvalidCertificates' r = _subprocess.call(command, shell=True) _events.fire('pytsite.mongodb@restore') _maintenance.disable() return r
def __init__(self, model: str, **kwargs): """Init """ if not model: raise RuntimeError('No model specified') if not odm_auth.check_model_permissions(model, [ PERM_CREATE, PERM_MODIFY, PERM_DELETE, PERM_MODIFY_OWN, PERM_DELETE_OWN ]): raise errors.ForbidOperation( "Current user is not allowed to browse '{}' entities".format( model)) # Model self._model = model # Model class self._model_class = _api.get_model_class(self._model) self._current_user = auth.get_current_user() self._browse_rule = kwargs.get('browse_rule', self._model_class.odm_ui_browse_rule()) self._m_form_rule = kwargs.get('m_form_rule', self._model_class.odm_ui_m_form_rule()) self._d_form_rule = kwargs.get('d_form_rule', self._model_class.odm_ui_d_form_rule()) # Widget widget_class = self._model_class.odm_ui_browser_widget_class() if not (issubclass(widget_class, widget.misc.DataTable)): raise TypeError('Subclass of {} expected, got'.format( widget.misc.DataTable, widget_class)) self._widget = widget_class(uid='odm-ui-browser-' + model, rows_url=http_api.url( 'odm_ui@get_browser_rows', { 'model': self._model, 'browse_rule': self._browse_rule, 'm_form_rule': self._m_form_rule, 'd_form_rule': self._d_form_rule, }), update_rows_url=http_api.url( 'odm_ui@put_browser_rows', {'model': model})) # Call model's class to perform setup tasks _api.dispense_entity(self._model).odm_ui_browser_setup(self) # Notify external events listeners events.fire('odm_ui@browser_setup.{}'.format(self._model), browser=self) # Check if the model specified data fields if not self.data_fields: raise RuntimeError('No data fields was defined') # Actions column if self._model_class.odm_ui_entity_actions_enabled() and \ (self._model_class.odm_ui_modification_allowed() or self._model_class.odm_ui_deletion_allowed()): self.insert_data_field('entity-actions', 'odm_ui@actions', False)
def register_model(model: str, cls: Union[str, Type[_model.Entity]], replace: bool = False): """Register a new ODM model """ if isinstance(cls, str): cls = util.get_module_attr(cls) # type: Type[_model.Entity] if not issubclass(cls, _model.Entity): raise TypeError("Unable to register model '{}': subclass of odm.model.Entity expected." .format(model)) if is_model_registered(model) and not replace: raise _error.ModelAlreadyRegistered(model) # Create finder cache pool for each newly registered model if not replace: cache.create_pool('odm.finder.' + model) _MODEL_TO_CLASS[model] = cls cls.on_register(model) events.fire('*****@*****.**', model=model, cls=cls, replace=replace) mock = dispense(model) # Save model's collection name _MODEL_TO_COLLECTION[model] = mock.collection _COLLECTION_NAME_TO_MODEL[mock.collection.name] = model # Automatically create indices on new collections if mock.collection.name not in mongodb.get_collection_names(): mock.create_indexes()
def sign_in(auth_driver_name: str = None, data: dict = None) -> _model.AbstractUser: """Authenticate user """ # Get user from driver user = get_auth_driver(auth_driver_name).sign_in(data) if user.status != USER_STATUS_ACTIVE: raise _error.UserNotActive() if is_sign_up_confirmation_required() and not (user.is_confirmed or user.is_admin): raise _error.UserNotConfirmed() switch_user(user) # Update statistics user.sign_in_count += 1 user.last_sign_in = datetime.now() user.save() # Login event events.fire('auth@sign_in', user=user) return user
def _on_setup_widgets(self): from ._api import dispense_entity model = self.attr('model') eid = self.attr('eid') # Setting up form's widgets through entity hook and global event entity = dispense_entity(model, eid) entity.odm_ui_m_form_setup_widgets(self) events.fire('odm_ui@m_form_setup_widgets.{}'.format(model), frm=self, entity=entity) if self.current_step == 1: # Entity model self.add_widget( widget.input.Hidden( uid='model', value=model, form_area='hidden', )) # Entity ID self.add_widget( widget.input.Hidden( uid='eid', value=eid, form_area='hidden', )) # Entity ref self.add_widget( widget.input.Hidden( uid='ref', value=entity.ref if not entity.is_new else None, form_area='hidden', )) # Cancel button URL cancel_href = self.redirect if not cancel_href or cancel_href == 'ENTITY_VIEW': if self.referer != self.location and self.referer: cancel_href = self.referer elif not entity.is_new and entity.odm_ui_view_url(): cancel_href = entity.odm_ui_view_url() else: cancel_href = router.base_url() # Cancel button self.add_widget( widget.button.Link( uid='action_cancel_' + str(self.current_step), weight=150, value=lang.t('odm_ui@cancel'), icon='fa fas fa-fw fa-remove fa-times', href=cancel_href, form_area='footer', ))
def exec(self): del self.args['_pytsite_router_rule_name'] endpoint = '/' + self.args.pop('http_api_endpoint') current_path = router.current_path(False) request_method = router.request().method # Switch language language = router.request().headers.get('Accept-Language') # type: str if language: for lng in language.split(','): lng = lng.strip() if not lng.startswith('q=') and lang.is_defined(language): lang.set_current(language) break try: events.fire('http_api@pre_request') rule = _api.match(router.request().method, endpoint) events.fire('http_api@request') controller = rule.controller_class() # type: routing.Controller controller.request = self.request controller.args.update(self.args) controller.args.update(rule.args) controller.args['_pytsite_http_api_rule_name'] = rule.name controller.args.validate() response = controller.exec() return response if isinstance( response, http.Response) else http.JSONResponse(response) except (http.error.Base, errors.ForbidOperation) as e: if reg.get('debug'): logger.error(e) else: logger.error('{} {}: {}'.format(request_method, current_path, e.description)) if isinstance(e, errors.ForbidOperation): e = http.error.Forbidden(e) if e.response and isinstance(e.response, http.JSONResponse): response = e.response response.status_code = e.code else: # It is important to do `str(e.description)`, because `e.description` might be an exception response = http.JSONResponse({'error': str(e.description)}, e.code) return response except UserWarning as e: logger.warn('{} {}: {}'.format(request_method, current_path, e)) return http.JSONResponse({'warning': str(e)}, 200) except Exception as e: logger.error('{} {}: {}'.format(request_method, current_path, e), exc_info=e) return http.JSONResponse({'error': str(e)}, 500)
def setup_widgets(self): """Setup widgets """ # 'Submit' button for the last step if self.steps == self._current_step and self._submit_button: self.add_widget(self._submit_button) # 'Next' button for all steps except the last one if self._current_step < self.steps: self.add_widget( _widget.button.Submit(weight=200, uid='action_forward_' + str(self._current_step + 1), value=_lang.t('form@forward'), form_area='footer', color='primary', css='form-action-forward', icon='fa fas fa-fw fa-forward', data={ 'to-step': self._current_step + 1, })) # 'Back' button for all steps except the first one if self._current_step > 1: self.add_widget( _widget.button.Button(weight=100, uid='action_backward_' + str(self._current_step - 1), value=_lang.t('form@backward'), form_area='footer', form_step=self._current_step, css='form-action-backward', icon='fa fas fa-fw fa-backward', data={ 'to-step': self._current_step - 1, })) # Ask form instance to setup widgets self._on_setup_widgets() # Ask others to setup form's widgets _events.fire('form@setup_widgets.' + self.name, frm=self) # Restore widgets' values if self._cache: try: for k, v in _cache.get_pool('form.form_values').get_hash( self._uid).items(): try: self.get_widget(k).set_val(v) except _error.WidgetNotExistError: pass except _cache.error.KeyNotExist: pass return self
def exec(self) -> dict: try: user = auth.get_user(uid=self.arg('uid')) jsonable = user.as_jsonable() events.fire('auth_http_api@get_user', user=user, json=jsonable) return jsonable except auth.error.UserNotFound: raise self.not_found()
def save(self): if self.is_anonymous: raise RuntimeError('Anonymous user cannot be saved') if self.is_system: raise RuntimeError('System user cannot be saved') events.fire('auth@user_pre_save', user=self) self.do_save() events.fire('auth@user_save', user=self) return self
def sign_up(auth_driver_name: str = None, data: dict = None) -> _model.AbstractUser: """Register a new user """ if not is_sign_up_enabled(): raise _error.SignupDisabled() user = get_auth_driver(auth_driver_name).sign_up(data) events.fire('auth@sign_up', user=user) return user
def define_permission(name: str, description: str, group: str): """Define a permission. """ if group not in _groups: raise _error.PermissionGroupNotDefined("Permission group '{}' is not defined.".format(group)) try: get_permission(name) raise _error.PermissionAlreadyDefined("Permission '{}' is already defined.".format(name)) except _error.PermissionNotDefined: _permissions.append((name, description, group)) events.fire('permission@define', name=name)
def delete(self): from . import _api # Check if the role is used by users user = _api.find_user(query.Query(query.Eq('roles', self))) if user: raise errors.ForbidDeletion(lang.t('role_used_by_user', {'role': self, 'user': user.login})) events.fire('auth@role_pre_delete', user=self) self.do_delete() events.fire('auth@role_delete', user=self) return self
def dump_all() -> str: """Dump all tags """ if not _tags: raise RuntimeError('reset() should be called before') _events.fire('pytsite.metatag@dump_all') r = str() for tag in _tags[_threading.get_id()]: r += dump(tag) + '\n' return r
def _do_reload(): """Modify 'touch.reload' file """ touch_reload_path = _path.join(_reg.get('paths.storage'), 'touch.reload') _events.fire('pytsite.reload@before_reload') with open(touch_reload_path, 'wt') as f: f.write(_util.w3c_datetime_str()) _events.fire('pytsite.reload@reload') _console.print_info(_lang.t('pytsite.reload@app_is_reloading'))
def delete(self): events.fire('auth@user_pre_delete', user=self) for user in self.follows: self.remove_follows(user) for user in self.blocked_users: self.remove_blocked_user(user) self.do_delete() events.fire('auth@user_delete', user=self) return self
def clear_cache(model: str): """Get finder cache pool """ try: # Clear finder cache cache.get_pool('odm.finder.' + model).clear() # Cleanup entities cache for k in _ENTITIES_CACHE.keys(): if k.startswith(model + '.'): _ENTITIES_CACHE.rm(k) events.fire('*****@*****.**', model=model) except cache.error.PoolNotExist: pass
def register_storage_driver(driver: _driver.Storage): """Register storage driver """ global _storage_driver if _storage_driver: raise _error.DriverRegistered('Storage driver is already registered') if not isinstance(driver, _driver.Storage): raise TypeError('Instance of {} expected'.format(type( _driver.Storage))) _storage_driver = driver events.fire('auth@register_storage_driver', driver=driver)
def _on_pre_save(self, **kwargs): """Hook """ super()._on_pre_save(**kwargs) if self.has_field('alias') and not self.alias: self.f_set('alias', self.f_get('title')) if self.is_new and self.has_field('order') and not self.order: from . import _api e = _api.find(self.model).eq('_parent', self.parent).sort([ ('order', odm.I_DESC) ]).first() self.order = ((int(ceil(e.order / 10.0)) * 10) + 10) if e else 10 events.fire('[email protected]_save', term=self)
def _split_msg_id(msg_id: str) -> list: """Split message ID into message ID and package name. """ for r in _events.fire('pytsite.lang@split_msg_id', msg_id=msg_id): msg_id = r return msg_id.split('@')[:2] if '@' in msg_id else ['app', msg_id]
def create_comment(thread_id: str, body: str, author: auth.model.AbstractUser, status: str = 'published', parent_uid: str = None, driver_name: str = None) -> _model.AbstractComment: """Create a new comment """ # Check min length if len(body) < get_comment_min_body_length(): raise _error.CommentTooShort(lang.t('comments@error_body_too_short')) # Check max length if len(body) > get_comment_max_body_length(): raise _error.CommentTooLong(lang.t('comments@error_body_too_long')) # Check status if status not in get_comment_statuses(): raise _error.InvalidCommentStatus( "'{}' is not a valid comment's status.".format(status)) # Load driver driver = get_driver(driver_name) # Create comment comment = driver.create_comment(thread_id, body, author, status, parent_uid) events.fire('comments@create_comment', comment=comment) # Send email notification about reply if reg.get('comments.email_notify', True) and comment.is_reply: parent_comment = get_comment(comment.parent_uid) if comment.author != parent_comment.author: tpl_name = 'comments@mail/{}/reply'.format(lang.get_current()) m_subject = lang.t('comments@mail_subject_new_reply') m_body = tpl.render( tpl_name, { 'reply': comment, 'comment': get_comment(comment.parent_uid, driver_name) }) m_from = '{} <{}>'.format(author.first_last_name, mail.mail_from()[1]) mail.Message(parent_comment.author.login, m_subject, m_body, m_from).send() return comment
def _on_setup_form(self): """Hook """ model = self.attr('model') if not model: raise ValueError('Model is not specified') if not self.name: self.name = 'odm_ui_modify_' + model try: from ._api import dispense_entity entity = dispense_entity(model, self.attr('eid')) except odm.error.EntityNotFound: raise http.error.NotFound() if entity.is_new: # Check if entities of this model can be created perms_allow = entity.odm_auth_check_entity_permissions(PERM_CREATE) odm_ui_allows = entity.odm_ui_creation_allowed() if not (perms_allow and odm_ui_allows): raise http.error.Forbidden() # Setup form title self.title = entity.t('odm_ui_form_title_create_' + model) else: # Check if the entity can be modified perms_allow = entity.odm_auth_check_entity_permissions(PERM_MODIFY) odm_ui_allows = entity.odm_ui_modification_allowed() if not (perms_allow and odm_ui_allows): raise http.error.Forbidden() # Setup form title self.title = entity.t('odm_ui_form_title_modify_' + model) # Setting up the form through entity hook and global event entity.odm_ui_m_form_setup(self) events.fire('odm_ui@m_form_setup.{}'.format(model), frm=self, entity=entity) # Redirect if not self.redirect: self.redirect = 'ENTITY_VIEW' # CSS self.css += ' odm-ui-form odm-ui-m-form odm-ui-form-' + model
def _on_after_save(self, first_save: bool = False, **kwargs): """Hook """ from . import _api # Recalculate tags weights if first_save and self.has_field('tags'): for t in self.tags: weight = 0 for model in _api.get_models().keys(): try: weight += _api.find(model, language=self.language).inc('tags', [t]).count() except odm.error.FieldNotDefined: pass try: auth.switch_user_to_system() t.f_set('weight', weight).save(fast=True) finally: auth.restore_user() # Update localization entities references # For each language except current one for lng in lang.langs(False): # Get localization ref for lng localization = self.f_get('localization_' + lng) # If localization is set if isinstance(localization, Content): # If localized entity hasn't reference to this entity, set it if localization.f_get('localization_' + self.language) != self: localization.f_set('localization_' + self.language, self).save() # If localization is not set elif localization is None: # Clear references from localized entities f = _api.find(self.model, language=lng).eq('localization_' + self.language, self) for referenced in f.get(): referenced.f_set('localization_' + self.language, None).save() # Notify content status change if self.has_field('status') and self.has_field('prev_status') and self.status != self.prev_status: self.content_on_status_change() events.fire('*****@*****.**', entity=self) events.fire('content@entity.{}.save'.format(self.model), entity=self)
def _on_pre_save(self, **kwargs): """Hook """ super()._on_pre_save(**kwargs) c_user = auth.get_current_user() # Content must be reviewed by moderator if self.has_field('status'): sts = self.content_statuses() if CONTENT_STATUS_WAITING in sts and CONTENT_STATUS_PUBLISHED in sts \ and self.status == CONTENT_STATUS_PUBLISHED \ and not self.odm_auth_check_entity_permissions(CONTENT_PERM_BYPASS_MODERATION): self.f_set('status', CONTENT_STATUS_WAITING) # Language is required if not self.language or not self.f_get('language_db'): self.f_set('language', lang.get_current()) # Author is required if self.has_field('author') and self.get_field('author').is_required and not self.author: if not c_user.is_anonymous: self.f_set('author', c_user) else: raise RuntimeError('Cannot assign author, because current user is anonymous') # Extract inline images from the body if self.has_field('body') and self.has_field('images'): body, images = _extract_images(self) # If new images has been extracted if images: self.f_set('body', body) self.f_set('images', list(self.images) + images) # Extract inline videos from the body if self.has_field('body') and self.has_field('video_links'): body, video_links = _extract_video_links(self) # If new video links has been extracted if video_links: self.f_set('body', body) self.f_set('video_links', list(self.video_links) + video_links) events.fire('[email protected]_save', entity=self) events.fire('content@entity.{}.pre_save.'.format(self.model), entity=self)
def t(msg_id: str, args: dict = None, language: str = None, exceptions: bool = False, use_fallback: bool = True) -> str: """Translate a message """ global _globals if not language: language = get_current() if language not in _languages: raise _error.LanguageNotSupported("Language '{}' is not supported".format(language)) if msg_id in _globals: return _globals[msg_id](language, args) # Determining package name and message ID package_name, msg_id = _split_msg_id(msg_id) # Try to get message translation string from cache cache_key = '{}-{}@{}'.format(language, package_name, msg_id) msg = _translated_strings_cache.get(cache_key) # Message translation is not found in cache, try to fetch it if not msg: # Try to get translation via event for r in _events.fire('pytsite.lang@translate', language=language, package_name=package_name, msg_id=msg_id): msg = r # Load translation from package's data if not msg: lang_file_content = get_package_translations(package_name, language) if msg_id not in lang_file_content: # Searching for fallback translation fallback = get_fallback() if use_fallback and fallback != language: return t(package_name + '@' + msg_id, args, fallback, exceptions, False) else: if exceptions: raise _error.TranslationError( "Translation is not found for '{}@{}'".format(package_name, msg_id)) else: return package_name + '@' + msg_id msg = lang_file_content[msg_id] # Cache translation string _translated_strings_cache[cache_key] = msg # Replace placeholders if args: for k, v in args.items(): msg = msg.replace(':' + str(k), str(v)) # Replace sub-translations msg = _SUB_TRANS_TOKEN_RE.sub(lambda match: t(match.group(1)), msg) return msg
def _split_location(location: str) -> Tuple[str, str]: """Split asset path into package name and asset path """ for r in events.fire('assetman@split_location', location=location): location = r package_name, assets_path = location.split( '@')[:2] if '@' in location else ['app', location] return resolve_package(package_name), assets_path
def create_user(login: str, password: str = None) -> _model.AbstractUser: """Create a new user """ if not login: raise _error.UserCreateError(lang.t('auth@login_str_rules')) # Various checks if login not in (_model.ANONYMOUS_USER_LOGIN, _model.SYSTEM_USER_LOGIN): try: # Check user existence get_user(login) raise _error.UserExists() except _error.UserNotFound: # Check user login try: user_login_rule.validate(login) except validation.error.RuleError as e: raise _error.UserCreateError(e) # Create user user = get_storage_driver().create_user(login, password) # Attach roles if login not in (_model.ANONYMOUS_USER_LOGIN, _model.SYSTEM_USER_LOGIN): # Set user's status user.status = get_new_user_status() # Generate confirmation hash if is_sign_up_confirmation_required(): user.is_confirmed = False user.roles = [get_role(r) for r in get_new_user_roles()] user.save() events.fire('auth@user_create', user=user) else: user.status = USER_STATUS_ACTIVE user.roles = [get_role('anonymous')] return user
def sign_out(user: _model.AbstractUser): """Sign out a user """ # Anonymous user cannot be signed out if user.is_anonymous: return try: # All operation on current user perform on behalf of system user switch_user_to_system() # Ask drivers to perform necessary operations for driver in _authentication_drivers.values(): driver.sign_out(user) # Notify listeners events.fire('auth@sign_out', user=user) finally: # Set anonymous user as current switch_user_to_anonymous()
def __init__(self, model: str, obj_id: Union[str, ObjectId, None] = None): """Init """ # Define collection name if it wasn't specified if not self._collection_name: self._collection_name = lang.english_plural(model) self._model = model self._is_new = True self._is_modified = True self._is_being_saved = False self._is_being_deleted = False self._is_deleted = False self._indexes = [] self._has_text_index = False self._pending_children = [] self._fields = OrderedDict() # type: Dict[str, _field.Base] # Define 'system' fields self.define_field(_field.ObjectId('_id', is_required=True)) self.define_field(_field.String('_ref', is_required=True)) self.define_field(_field.String('_model', is_required=True, default=self._model)) self.define_field(_field.Ref('_parent', model=model)) self.define_field(_field.Integer('_depth')) self.define_field(_field.DateTime('_created', default=datetime.now())) self.define_field(_field.DateTime('_modified', default=datetime.now())) # Define field to store changes of other fields if self._history_fields: self.define_field(_field.List('_history')) # Setup fields self._setup_fields() events.fire('[email protected]_fields', entity=self) events.fire('[email protected]_fields.{}'.format(model), entity=self) # Delegate indexes setup process to the hook method self.define_index([('_ref', I_ASC)]) self.define_index([('_parent', I_ASC)]) self.define_index([('_created', I_ASC)]) self.define_index([('_modified', I_ASC)]) self._setup_indexes() events.fire('[email protected]_indexes', entity=self) events.fire('[email protected]_indexes.{}'.format(model), entity=self) # Load fields data from database or cache if obj_id: self._load_fields_data(ObjectId(obj_id) if isinstance(obj_id, str) else obj_id)
def cron_every_min(): out = '' for r in _events.fire('pytsite.stats@update'): if not r or not isinstance(r, str): continue out += '- ' + r + '\n' with open(_path.join(_reg.get('paths.storage'), 'stats.txt'), 'wt') as f: f.write(_util.w3c_datetime_str() + '\n\n' + out) if out: _logger.info('Current stats:\n{}'.format(out[:-1]))
def exec(self) -> list: uids = self.arg('uids') exclude = self.arg('exclude') search = self.arg('search') or self.arg('q') r = [] if uids: for uid in self.arg('uids'): try: user = auth.get_user(uid=uid) json = user.as_jsonable() events.fire('auth_http_api@get_user', user=user, json=json) r.append(json) except Exception as e: # Any exception is ignored due to safety reasons logger.warn(e) elif search and reg.get('auth_http_api.search', False): q = query.Query() q.add( query.Or([ query.Regex('first_name', '^{}'.format(search), True), query.Regex('last_name', '^{}'.format(search), True), ])) if not auth.get_current_user().is_admin: q.add(query.Eq('is_public', True)) if exclude: q.add(query.Nin('uid', exclude)) for user in auth.find_users(q, limit=self.arg('limit'), skip=self.arg('skip')): r.append(user.as_jsonable()) return r
def _on_setup_widgets(self): """Hook """ setting_uid = self.attr('setting_uid') events.fire('[email protected]_widgets', frm=self) events.fire('[email protected]_widgets.' + setting_uid, frm=self) # Fill form widgets with values for k, v in reg.get(setting_uid, {}).items(): try: self.get_widget('setting_' + k).value = v except form.WidgetNotExistError: pass # "Cancel" button self.add_widget(widget.button.Link( uid='action_cancel_' + str(self.current_step), weight=100, value=lang.t('settings@cancel'), icon='fa fas fa-fw fa-ban', href=router.rule_url('admin@dashboard'), form_area='footer', ))
def exec(self) -> dict: user = auth.get_current_user() # Check permissions if user.is_anonymous or (user.uid != self.arg('uid') and not user.is_admin): raise self.forbidden() allowed_fields = ('email', 'nickname', 'picture', 'first_name', 'last_name', 'description', 'birth_date', 'gender', 'phone', 'country', 'city', 'urls', 'is_public') for k, v in self.args.items(): if k in allowed_fields: user.set_field(k, v) if user.is_modified: user.save() json = user.as_jsonable() events.fire('auth_http_api@get_user', user=user, json=json) return json
def cron_1min(): """pytsite.cron.1min """ global _working if _working: _logger.warn('Content import is still working') return _working = True max_errors = _reg.get('content_import.max_errors', 13) max_items = _reg.get('content_import.max_items', 10) delay_errors = _reg.get('content_import.delay_errors', 120) importer_finder = _odm.find('content_import') \ .eq('enabled', True) \ .lt('paused_till', _datetime.now()) \ .sort([('errors', _odm.I_ASC)]) for importer in importer_finder.get(): # type: _model.ContentImport options = dict(importer.driver_opts) options.update({ 'content_author': importer.content_author, 'content_model': importer.content_model, 'content_language': importer.content_language, 'content_status': importer.content_status, 'content_section': importer.content_section, }) driver = _api.get_driver(importer.driver) items_imported = 0 try: _logger.info('Content import started. Driver: {}. Options: {}'.format(driver.get_name(), options)) # Get entities from driver and save them for entity in driver.get_entities(_frozendict(options)): if items_imported == max_items: break try: # Append additional tags if entity.has_field('tags'): for tag_title in importer.add_tags: tag = _tag.find_by_title(tag_title, language=importer.content_language) if not tag: tag = _tag.dispense(tag_title, language=importer.content_language).save() entity.f_add('tags', tag) # Save entity entity.save() # Notify listeners _events.fire('content_import@import', driver=driver, entity=entity) _logger.info("Content entity imported: '{}'".format(entity.f_get('title'))) items_imported += 1 # Entity was not successfully saved; make record in the log and skip to the next entity except Exception as e: # Delete already attached images to free space if entity.has_field('images') and entity.images: for img in entity.images: img.delete() _logger.error("Error while creating entity '{}'. {}".format(entity.title, str(e)), exc_info=e) # Mark that driver made its work without errors importer.f_set('errors', 0) _logger.info('Content import finished. Entities imported: {}.'.format(items_imported)) except Exception as e: # Increment errors counter importer.f_inc('errors') # Store info about error importer.f_set('last_error', str(e)) if importer.errors >= max_errors: # Disable if maximum errors count reached importer.f_set('enabled', False) else: # Pause importer importer.f_set('paused_till', _datetime.now() + _timedelta(minutes=delay_errors)) _logger.error(e) # Continue to the next importer continue finally: importer.save() _working = False
def _init(): """Init wrapper """ import sys from importlib import import_module from sys import argv, exit from os import path, environ from getpass import getuser from socket import gethostname from . import reg, package_info, semver # Load regisrty memory driver reg.set_driver(reg.driver.Memory()) # Environment type and name reg.put('env.name', getuser() + '@' + gethostname()) if len(argv) > 1 and argv[1] == 'test': reg.put('env.type', 'testing') else: reg.put('env.type', 'wsgi' if 'UWSGI_ORIGINAL_PROC_NAME' in environ else 'console') # Detect application's root directory path cur_dir = path.abspath(path.dirname(__file__)) while True: if path.exists(path.join(cur_dir, 'app', 'app.json')): root_path = cur_dir break elif cur_dir != '/': cur_dir = path.abspath(path.join(cur_dir, path.pardir)) else: raise RuntimeError('Cannot determine root directory of application') # It is important for correct importing of packages inside 'themes', 'plugins', etc sys.path.append(root_path) # Base filesystem paths env_path = environ.get('VIRTUAL_ENV', path.join(root_path, 'env')) app_path = path.join(root_path, 'app') reg.put('paths.root', root_path) reg.put('paths.env', env_path) reg.put('paths.app', app_path) for n in ['config', 'log', 'static', 'storage', 'tmp']: reg.put('paths.' + n, path.join(root_path, n)) # PytSite path reg.put('paths.pytsite', path.join(root_path, path.dirname(__file__))) # uWSGI does not export virtualenv paths, do it by ourselves if 'VIRTUAL_ENV' not in environ: environ['VIRTUAL_ENV'] = env_path environ['PATH'] = path.join(env_path, 'bin') + ':' + environ['PATH'] # Additional filesystem paths reg.put('paths.session', path.join(reg.get('paths.storage'), 'session')) # Debug is disabled by default reg.put('debug', False) # Check for 'app' package if not path.exists(app_path): raise FileNotFoundError("Directory '{}' is not found".format(app_path)) # Switch registry to the file driver file_driver = reg.driver.File(reg.get('paths.config'), reg.get('env.name'), reg.get_driver()) reg.set_driver(file_driver) # Default output parameters reg.put('output', { 'minify': not reg.get('debug'), }) # Initialize logger from . import logger logger.info('') logger.info('---===[ PytSite-{} Started ]===---'.format(package_info.version('pytsite'))) # Initialize rest of the system from pytsite import console, util try: # Initialize cache with default driver from pytsite import cache cache.set_driver(cache.driver.File()) # Load required core packages, order is important for pkg_name in ('cron', 'stats', 'reload', 'update', 'plugman', 'testing'): import_module('pytsite.' + pkg_name) # Register app's resources if path.exists(path.join(app_path, 'res', 'lang')): from pytsite import lang lang.register_package('app') if path.exists(path.join(app_path, 'res', 'tpl')): from pytsite import tpl tpl.register_package('app') # Load app package from pytsite import plugman, events try: import app package_info.check_requirements('app') # app_load() hook if hasattr(app, 'app_load'): app.app_load() # app_load_{env.type}() hook hook_name = 'app_load_{}'.format(reg.get('env.type')) if hasattr(app, hook_name): getattr(app, hook_name)() events.fire('pytsite.app_load') logger.debug('Application loaded') except Exception as e: logger.error(e) console.print_warning('Application load error: {}'.format(e)) finally: events.fire('pytsite.load') logger.debug('PytSite initialized and ready to work') except Warning as e: console.print_warning(e) except Exception as e: console.print_error(e) if reg.get('debug'): raise e exit(1)
def exec(self) -> dict: events.fire('comments@report', uid=self.arg('uid')) return {'status': True}
def _on_pre_delete(self, **kwargs): """Hook """ super()._on_pre_delete(**kwargs) events.fire('[email protected]_delete', term=self)
def exec(self, args: tuple = (), **kwargs): """Execute the command. """ app_path = _reg.get('paths.app') config_path = _reg.get('paths.config') stage = self.opt('stage') stop_after = self.opt('stop-after') _chdir(app_path) _maintenance.enable() d = self._get_update_data() if not d['update_in_progress']: d['pytsite_version_from'] = str(_package_info.version('pytsite')) d['app_version_from'] = str(_package_info.version('app')) d['update_in_progress'] = True self._set_update_data(d) # Stage 1: update pip and PytSite if stage == 1: _console.print_info(_lang.t('pytsite.update@updating_environment')) # Update pip _pip.install('pip', None, True, self.opt('debug')) # Update PytSite _pip.install('pytsite', _package_info.requires_pytsite('app'), True, self.opt('debug')) d['pytsite_version_to'] = str(_package_info.version('pytsite', False)) self._set_update_data(d) # Notify listeners _events.fire('pytsite.update@stage_1') if stop_after != 1: _subprocess.call(['./console', 'update', '--stage=2']) else: _maintenance.disable() # Stage 2: update application and configuration elif stage == 2: # Notify listeners about PytSite update _events.fire('pytsite.update@pytsite', v_from=_semver.Version(d['pytsite_version_from'])) # Update configuration if _path.exists(_path.join(config_path, '.git')): _console.print_info(_lang.t('pytsite.update@updating_configuration')) _subprocess_run(['git', '-C', config_path, 'pull']) # Update application if _path.exists(_path.join(app_path, '.git')): _console.print_info(_lang.t('pytsite.update@updating_application')) _subprocess_run(['git', '-C', app_path, 'pull']) d['app_version_to'] = str(_package_info.version('app', False)) self._set_update_data(d) # Notify listeners _events.fire('pytsite.update@stage_2') if stop_after != 2: _subprocess.call(['./console', 'update', '--stage=3']) else: _maintenance.disable() # Stage 3: finish update process elif stage == 3: _console.print_info(_lang.t('pytsite.update@applying_updates')) # Notify listeners about application update _events.fire('pytsite.update@app', v_from=_semver.Version(d['app_version_from'])) # Notify listeners _events.fire('pytsite.update@update') # Application's update hook import app if hasattr(app, 'app_update') and callable(app.app_update): app.app_update(v_from=_semver.Version(d['app_version_from'])) # Mark that update was finished successfully d['update_in_progress'] = False self._set_update_data(d) # Disable maintenance mode _maintenance.disable() # Reload the application _reload.reload()
def __init__(self, msg: str = None, **kwargs): self._msg = msg events.fire('auth@sign_up_error', exception=self, data=kwargs.get('data'))
def on_pytsite_load(): update_info = _api.get_update_info() if not update_info: return # If there waiting updates exist, reload the application if _reg.get('env.type') == 'wsgi': _logger.warn('Application needs to be loaded in console to finish plugins update') return failed_plugins = [] # Call 'plugin_pre_install()' hooks for p_name, info in update_info.items(): v_to = _semver.Version(info['version_to']) try: # Check if the plugin is installed and loaded plugin = _api.get(p_name) # Call plugin_pre_install() hook if hasattr(plugin, 'plugin_pre_install') and callable(plugin.plugin_pre_install): plugin.plugin_pre_install() # Fire 'pre_install' event _events.fire('pytsite.plugman@pre_install', name=p_name, version=v_to) except _error.PluginNotLoaded as e: _logger.error(e) _console.print_warning(_lang.t('pytsite.plugman@plugin_install_error', { 'plugin': p_name, 'version': v_to, 'msg': str(e), })) failed_plugins.append(p_name) continue # Finish installing/updating plugins for p_name, info in update_info.items(): if p_name in failed_plugins: continue plugin = _api.get(p_name) v_from = _semver.Version(info['version_from']) v_to = _semver.Version(info['version_to']) try: _logger.info(_lang.t('pytsite.plugman@installing_plugin', { 'plugin': p_name, 'version': v_to, })) # Call plugin_install() hook if hasattr(plugin, 'plugin_install') and callable(plugin.plugin_install): plugin.plugin_install() # Fire 'install' event _events.fire('pytsite.plugman@install', name=p_name, version=v_to) _console.print_success(_lang.t('pytsite.plugman@plugin_install_success', { 'plugin': p_name, 'version': v_to, })) except Exception as e: _logger.error(e) _console.print_warning(_lang.t('pytsite.plugman@plugin_install_error', { 'plugin': p_name, 'version': v_to, 'msg': str(e), })) continue # Update plugin if v_from != '0.0.0': try: _console.print_info(_lang.t('pytsite.plugman@updating_plugin', { 'plugin': p_name, 'v_from': v_from, 'v_to': v_to, })) # Call plugin_update() hook if hasattr(plugin, 'plugin_update') and callable(plugin.plugin_update): plugin.plugin_update(v_from=v_from) # Fire 'update' event _events.fire('pytsite.plugman@update', name=p_name, v_from=v_from) _console.print_success(_lang.t('pytsite.plugman@plugin_update_success', { 'plugin': p_name, 'version': v_to, })) except Exception as e: _console.print_warning(_lang.t('pytsite.plugman@plugin_update_error', { 'plugin': p_name, 'version': v_to, 'msg': str(e), })) continue # Remove info from update queue _api.rm_update_info(p_name)
def cron_hourly(): _cleanup_tmp_files() _events.fire('pytsite.cleanup@cleanup')