def ensure_in_session(item): if not object_session(item): try: constants.db_session().add(item) return item except exc.InvalidRequestError: return constants.db_session().merge(item)
def _generate_and_add(self, img_hash, generate, cover_path, model, item_id, image_size): sess = constants.db_session() model_name = db.model_name(model) new = False if cover_path: self.cover = sess.query( db.Profile).filter(db.Profile.data == img_hash).one_or_none() else: self.cover = db.Profile() new = True if generate: with self.generate.call_capture(model_name, model_name, item_id, image_size) as plg: self.cover.path = plg.first() self.cover.data = img_hash self.cover.size = str(tuple(image_size)) if self.cover.path and generate: if new: s = constants.db_session() i = s.query(model).get(item_id) i.profiles.append(self.cover) sess.commit() elif not self.cover.path: self.cover = None return self.cover
def _calculate(self, gallery_or_id, all_gallery_tags={}): assert isinstance(gallery_or_id, (str, int, db.Gallery)) sess = constants.db_session() with db.no_autoflush(sess): data = {} g_id = gallery_or_id.id if isinstance( gallery_or_id, db.Gallery) else gallery_or_id g_id = str(g_id) # because JSON keys are str tag_count = 0 tag_count_minimum = 3 if g_id in all_gallery_tags: g_tags = all_gallery_tags[g_id] for a, b in g_tags.items(): tag_count += len(b) self.set_max_progress(len(g_tags) + 3) else: if isinstance(gallery_or_id, db.Gallery): g_tags = gallery_or_id else: g_tags = database_cmd.GetModelItems().run( db.Taggable, join=db.Gallery.taggable, filter=db.Gallery.id == int(g_id)) if g_tags: g_tags = g_tags[0] tag_count = g_tags.tags.count() if tag_count > tag_count_minimum: g_tags = g_tags.compact_tags(g_tags.tags.all()) self.next_progress() if g_tags and tag_count > tag_count_minimum: log.d("Calculating similarity") g_tags = self._get_set(g_tags) data[g_id] = gl_data = {} update_dict = not all_gallery_tags max_prog = 3 for t_id, t in all_gallery_tags.items( ) or constants.db_session().query( db.Gallery.id, db.Taggable).join(db.Gallery.taggable): t_id = str(t_id) self.next_progress() if update_dict: all_gallery_tags[t_id] = t.compact_tags(t.tags.all()) max_prog += 1 self.set_max_progress(max_prog) if t_id == g_id: continue t_tags = self._get_set(all_gallery_tags[t_id]) if (math.sqrt(len(g_tags)) * math.sqrt(len(t_tags))) != 0: cos = len(g_tags & t_tags) / (math.sqrt( len(g_tags))) * math.sqrt(len(t_tags)) else: cos = 0 if cos: gl_data[t_id] = cos log.d("Finished calculating similarity") self.next_progress() return data
def exists(self, obj=False, strict=False): "obj: queries for the full object and returns it" if not constants.db_session: return self sess = constants.db_session() if obj: if strict: e = sess.query( self.__class__).filter_by(name=self.name).scalar() else: e = sess.query(self.__class__).filter( self.__class__.name.ilike("%{}%".format( self.name))).scalar() if not e: e = self else: if strict: e = sess.query(self.__class__.id).filter_by( name=self.name).scalar() is not None else: e = sess.query(self.__class__.id).filter( self.__class__.name.ilike("%{}%".format( self.name))).scalar() is not None sess.close() return e
def get_context(self, user=None, password=None): "Creates or retrieves existing context object for this client" s = constants.db_session() user_obj = None if user or password: log.d("Client provided credentials, authenticating...") if user == constants.super_user_name and not config.disable_default_user.value: log.d("Authenticating with default user") user_obj = s.query(db.User).filter( db.User.role == db.User.Role.default).one() else: user_obj = s.query( db.User).filter(db.User.name == user).one_or_none() if not user_obj or (not user_obj.role == db.User.Role.guest and not user_obj.password == password): raise exceptions.AuthWrongCredentialsError( utils.this_function(), "Wrong credentials") else: log.d("Client did not provide credentials") if not config.allow_guests.value: log.d("Guests are disallowed on this server") raise exceptions.AuthRequiredError( utils.this_function(), "Authentication is required") log.d("Authencticating as guest") user_obj = db.User(role=db.User.Role.guest) self.context['user'] = user_obj self.context['adresss'] = self._address if not self.context['user'].context_id: self.context['user'].context_id = uuid.uuid4().hex self.context['config'] = {} log.d("Client accepted") self._accepted = True
def main(self, model: db.Base, limit: int = 999, filter: str = "", order_by: str = "", offset: int = 0, join: str = "") -> tuple: s = constants.db_session() q = s.query(model) if join: if not isinstance(join, (list, tuple)): join = [join] for j in join: if isinstance(j, str): q = q.join(db.sa_text(j)) else: q = q.join(j) if filter: if isinstance(filter, str): q = q.filter(db.sa_text(filter)) else: q = q.filter(filter) if order_by: q = q.order_by(db.sa_text(order_by)) self.fetched_items = tuple(self._query(q, limit, offset)) self.fetched.emit(db.model_name(model), self.fetched_items) return self.fetched_items
def _match_namemixin(parent_model, child_model, term, options, capture=[ db.model_name(x) for x in _models() if issubclass(x, (db.NameMixin, db.Url)) ]): get_model = database_cmd.GetModelClass() parent_model = get_model.run(parent_model) child_model = get_model.run(child_model) match_string = PartialModelFilter._match_string_column term = ParseTerm().run(term) ids = set() if term.namespace.lower() == child_model.__name__.lower( ) or not term.namespace: col_on_parent = db.relationship_column(parent_model, child_model) s = constants.db_session() q = s.query(parent_model.id) if col_on_parent: q = q.join(col_on_parent) ids.update(x[0] for x in q.filter( match_string(child_model.name, term.tag, options)).all()) return ids
def create_user(role, name=None, password=None): """ Create user role: - default -- create a default user with no password if it doesnt exist - admin -- create admin user - user -- create regular user """ assert isinstance(role, User.Role) s = constants.db_session() if role == User.Role.default: if not s.query(User).filter(User.name == 'default').one_or_none(): s.add(User(name='default', role=User.Role.default)) s.commit() elif role == User.Role.user: pass elif role == User.Role.admin: s.add( User(name=name, password=PasswordHash.new(password, 15), role=User.Role.admin)) s.commit()
def get_related_count(item_type: enums.ItemType = enums.ItemType.Gallery, item_id: int = 0, related_type: enums.ItemType = enums.ItemType.Page): """ Get count of items related to given item Args: item_type: parent item item_id: id of parent item related_type: child item Returns: ``` { 'id': int, 'count': int } ``` """ item_type = enums.ItemType.get(item_type) related_type = enums.ItemType.get(related_type) _, parent_model = item_type._msg_and_model() child_msg, child_model = related_type._msg_and_model() col = db.relationship_column(parent_model, child_model) if not col: raise exceptions.APIError( utils.this_function(), "{} has no relationship with {}".format(related_type, item_type)) s = constants.db_session() count = s.query( child_model.id).join(col).filter(parent_model.id == item_id).count() return message.Identity('count', {'id': item_id, 'count': count})
def _match_tags(parent_model, child_model, term, options, capture=db.model_name(db.Taggable)): get_model = database_cmd.GetModelClass() parent_model = get_model.run(parent_model) child_model = get_model.run(child_model) match_string = PartialModelFilter._match_string_column term = ParseTerm().run(term) ids = set() col_on_parent = db.relationship_column(parent_model, child_model) col_on_child = db.relationship_column(child_model, db.NamespaceTags) col_tag = db.relationship_column(db.NamespaceTags, db.Tag) s = constants.db_session() q = s.query(parent_model.id) if term.namespace: col_ns = db.relationship_column(db.NamespaceTags, db.Namespace) items = q.join(col_on_parent).join(col_on_child).join(col_ns).join( col_tag).filter( db.and_op( match_string(db.Namespace.name, term.namespace, options, whole=True), match_string(db.Tag.name, term.tag, options))).all() else: items = q.join(col_on_parent).join(col_on_child).join( col_tag).filter(match_string(db.Tag.name, term.tag, options)).all() ids.update(x[0] for x in items) return ids
def exists(self, obj=False, strict=False): """Checks if gallery exists by path Params: obj -- queries for the full object and returns it """ e = None g = self.__class__ if self.path: head, tail = os.path.split(self.path) p, ext = os.path.splitext(tail if tail else head) sess = constants.db_session() if self.in_archive: head, tail = os.path.split(self.path_in_archive) p_a = tail if tail else head e = sess.query(self.__class__.id).filter( and_(g.path.ilike("%{}%".format(p)), g.path_in_archive.ilike( "%{}%".format(p_a)))).scalar() else: e = sess.query(self.__class__.id).filter( and_(g.path.ilike("%{}%".format(p)))).scalar() sess.close() if not obj: e = e is not None else: log_w( "Could not query for gallery existence because no path was set." ) return e
def _match_gallery(parent_model, child_model, term, options, capture=db.model_name(db.Gallery)): get_model = database_cmd.GetModelClass() parent_model = get_model.run(parent_model) child_model = get_model.run(child_model) match_string = PartialModelFilter._match_string_column match_int = PartialModelFilter._match_integer_column term = ParseTerm().run(term) ids = set() s = constants.db_session() if term.namespace: lower_ns = term.namespace.lower() if lower_ns == 'path': ids.update(x[0] for x in s.query(parent_model.id).filter( match_string(db.Gallery.path, term, options)).all()) elif lower_ns in ("rating", "stars"): ids.update(x[0] for x in s.query(parent_model.id).filter( match_int(db.Gallery.rating, term, options)).all()) return ids
def _match_parody(parent_model, child_model, term, options, capture=db.model_name(db.Parody)): get_model = database_cmd.GetModelClass() parent_model = get_model.run(parent_model) child_model = get_model.run(child_model) match_string = PartialModelFilter._match_string_column term = ParseTerm().run(term) ids = set() if term.namespace.lower() == 'parody' or not term.namespace: col_on_parent = db.relationship_column(parent_model, child_model) s = constants.db_session() q = s.query(parent_model.id) if col_on_parent: q = q.join(col_on_parent) ids.update( x[0] for x in q.join( child_model.names).filter( match_string( db.ParodyName.name, term.tag, options)).all()) return ids
def mapping_exists(self): sess = constants.db_session() e = sess.query(self.__class__).filter_by( and_(tag_id=self.tag_id, namespace_id=self.namespace_id)).scalar() if not e: e = self sess.close() return e
def exists(self, obj=False): """Checks if page exist by path Params: obj -- queries for the full object and returns it """ sess = constants.db_session() e = None p = self.__class__ if self.path: sess = constants.db_session() e = sess.query(p.id).filter( and_(p.path.ilike("%{}%".format(self.path)))).scalar() sess.close() if not obj: e = e is not None else: log_w( "Could not query for page existence because no path was set.") return e
def main(self, model: db.Base, item_id: int, image_size: enums.ImageSize) -> db.Profile: self.model = model if image_size == enums.ImageSize.Original: image_size = utils.ImageSize(0, 0) else: image_size = utils.ImageSize( *constants.image_sizes[image_size.name.lower()]) with self.models.call() as plg: for p in plg.all(default=True): self._supported_models.update(p) if self.model not in self._supported_models: raise exceptions.CommandError( utils.this_command(self), "Model '{}' is not supported".format(model)) img_hash = io_cmd.ImageItem.gen_hash(model, image_size, item_id) generate = True sess = constants.db_session() profile_size = str(tuple(image_size)) self.cover = sess.query(db.Profile).filter( db.and_op(db.Profile.data == img_hash, db.Profile.size == profile_size)).first() old_img_hash = None if self.cover: if io_cmd.CoreFS(self.cover.path).exists: generate = False else: old_img_hash = self.cover.data self.next_progress() if not generate: model_name = db.model_name(model) with self.invalidate.call_capture(model_name, model_name, item_id, image_size) as plg: if plg.first_or_default(): generate = True self.next_progress() if generate: constants.task_command.thumbnail_cleaner.wake_up() self.cover = self.run_native(self._generate_and_add, img_hash, old_img_hash, generate, model, item_id, image_size, profile_size).get() self.cover_event.emit(self.cover) return self.cover
def get_related_items( item_type: enums.ItemType = enums.ItemType.Gallery, item_id: int = 0, related_type: enums.ItemType = enums.ItemType.Page, limit: int = 100, offset: int = None, ): """ Get item related to given item Args: item_type: parent item item_id: id of parent item related_type: child item limit: limit the amount of items returned offset: offset the results by n items Returns: .. code-block:: guess [ related item message object, ... ] """ if not item_id: raise exceptions.APIError(utils.this_function(), "item_id must be a valid item id") item_type = enums.ItemType.get(item_type) related_type = enums.ItemType.get(related_type) _, parent_model = item_type._msg_and_model() child_msg, child_model = related_type._msg_and_model() col = db.relationship_column(parent_model, child_model) if not col: raise exceptions.APIError( utils.this_function(), "{} has no relationship with {}".format(related_type, item_type)) s = constants.db_session() q = s.query(child_model.id).join(col).filter(parent_model.id == item_id) if offset: q = q.offset(offset) item_ids = q.limit(limit).all() items = database_cmd.GetModelItems().run(child_model, {x[0] for x in item_ids}) item_list = message.List(db.model_name(child_model), child_msg) [item_list.append(child_msg(x)) for x in items] return item_list
def main(self, model: db.Base, term: str, match_options: dict = {}) -> typing.Set[int]: self.model = model model_name = db.model_name(self.model) self.term = term with self.models.call() as plg: for p in plg.all(default=True): self._supported_models.update(p) if self.model not in self._supported_models: raise exceptions.CommandError( utils.this_command(self), "Model '{}' is not supported".format(model)) options = get_search_options(match_options) log.d("Match options", options) related_models = db.related_classes(model) sess = constants.db_session() model_count = sess.query(model).count() with self.match_model.call_capture(model_name, model_name, model_name, self.term, options) as plg: for i in plg.all_or_default(): self.matched_ids.update(i) if len(self.matched_ids) == model_count: break has_all = False for m in related_models: if m in self._supported_models: with self.match_model.call_capture(db.model_name(m), model_name, db.model_name(m), self.term, options) as plg: for i in plg.all_or_default(): self.matched_ids.update(i) if len(self.matched_ids) == model_count: has_all = True break if has_all: break self.matched.emit(self.matched_ids) return self.matched_ids
def main(self, model: db.Base, item_id: int, image_size: enums.ImageSize) -> db.Profile: self.model = model if image_size == enums.ImageSize.Original: image_size = utils.ImageSize(0, 0) else: image_size = utils.ImageSize( *constants.image_sizes[image_size.name.lower()]) with self.models.call() as plg: for p in plg.all(default=True): self._supported_models.update(p) if self.model not in self._supported_models: raise exceptions.CommandError( utils.this_command(self), "Model '{}' is not supported".format(model)) img_hash = io_cmd.ImageItem.gen_hash(model, image_size, item_id) generate = True sess = constants.db_session() self.cover = sess.query( db.Profile).filter(db.Profile.data == img_hash).one_or_none() if self.cover: if io_cmd.CoreFS(self.cover.path).exists: generate = False else: self.cover = db.Profile() if generate: model_name = db.model_name(model) with self.generate.call_capture(model_name, model_name, item_id, image_size) as plg: self.cover.path = plg.first() self.cover.data = img_hash self.cover.size = str(tuple(image_size)) if self.cover.path and generate: i = GetModelItemByID().run(model, {item_id})[0] i.profiles.append(self.cover) sess.commit() elif not self.cover.path: self.cover = None self.cover_event.emit(self.cover) return self.cover
def main(self, model: db.Base, item_id: int, limit: int = 20) -> typing.Tuple[db.NamespaceTags]: assert issubclass(model, (db.Artist, db.Grouping, db.Collection)) s = constants.db_session() r = s.query(db.NamespaceTags).join(db.Taggable.tags).filter( db.Taggable.id.in_( s.query(db.Gallery.taggable_id).join(model.galleries).filter( model.id == item_id))).group_by(db.NamespaceTags).order_by( db.desc_expr(db.func.count( db.NamespaceTags.id))).limit(limit).all() return tuple(r)
def temporary_view( view_type: enums.TemporaryViewType = enums.TemporaryViewType. GalleryAddition, view_id: int = None, limit: int = 100, offset: int = 0, # sort_by: enums.ItemSort = None, # sort_desc: bool=False, ): """ not ready yet... Args: view_type: type of temporary view view_id: id of a specific view limit: amount of items per page offset: offset the results by n items Returns: .. code-block:: guess { 'items': [ ... ], 'count': int # count of all items in view } """ view_type = enums.TemporaryViewType.get(view_type) d = {'items': [], 'count': 0} msg_obj = None sess = constants.db_session() sess.autoflush = False if view_type == enums.TemporaryViewType.GalleryAddition: msg_obj = message.GalleryFS c = constants.store.galleryfs_addition.get({}) if view_id: c = list(c.get(view_id, tuple())) else: c = list(itertools.chain(*c.values())) d['count'] = len(c) d['items'] = [ msg_obj(x).json_friendly(False) if msg_obj else x for x in c[offset:offset + limit] ] return message.Identity('items', d)
def get_tags_count(): """ Get count of namespacetags in the database Returns: .. code-block:: guess { 'count' : int } """ s = constants.db_session() return message.Identity('count', {'count': s.query(db.NamespaceTags).count()})
def _generate_gallery_fs(self, found_paths, options): paths_len = len(found_paths) galleries = [] sess = constants.db_session() with db.no_autoflush(sess): for n, p in enumerate(found_paths, 1): self.next_progress(text=f"[{n}/{paths_len}] {p}") if options.get(config.skip_existing_galleries.fullname): if db.Gallery.exists_by_path(p): continue g = io_cmd.GalleryFS(p) if g.evaluate(): g.load_all() self.set_progress(text=f"[{n}/{paths_len}] {g.path.path} .. OK") galleries.append(g) return galleries
def get_context(self, user=None, password=None): "Creates or retrieves existing context object for this client" s = constants.db_session() user_obj = None if user or password: log.d("Client provided credentials, authenticating...") user_obj = s.query( db.User).filter(db.User.name == user).one_or_none() if user_obj: if not user_obj.password == password: raise exceptions.AuthError(utils.this_function(), "Wrong credentials") else: raise exceptions.AuthError(utils.this_function(), "Wrong credentials") else: log.d("Client did not provide credentials") if not constants.disable_default_user: log.d("Authenticating with default user") user_obj = s.query(db.User).filter( db.User.role == db.User.Role.default).one() else: if not constants.allow_guests: log.d("Guests are disallowed on this server") raise exceptions.AuthRequiredError(utils.this_function()) log.d("Authencticating as guest") user_obj = s.query(db.User).filter( db.and_op( db.User.address == self._ip, db.User.role == db.User.Role.guest)).one_or_none() if not user_obj: user_obj = db.User(role=db.User.Role.guest) s.add(user_obj) self.context = user_obj self.context.address = self._ip if not self.context.context_id: self.context.context_id = uuid.uuid4().hex self.context.config = None log.d("Client accepted") self._accepted = True s.commit()
def from_json(cls, msg, ignore_empty=True, skip_updating_existing=True, skip_descriptors=False, **kwargs): pref_title = msg.pop('preferred_title', False) obj = super().from_json(msg, ignore_empty, skip_updating_existing, skip_descriptors, **kwargs) with db.no_autoflush(db.object_session(obj) or constants.db_session()): if not skip_descriptors and pref_title: for mt in msg.get('titles', []): if utils.compare_json_dicts(mt, pref_title): break else: t = Title.from_json(pref_title, ignore_empty=ignore_empty, skip_updating_existing=skip_updating_existing, skip_descriptors=skip_descriptors, **kwargs) setattr(obj, 'preferred_title', t) if msg.get('pages'): obj.pages.reorder() if (msg.get('grouping') or msg.get('grouping_id')) and obj.grouping: obj.grouping.galleries.reorder() return obj
def get_count(item_type: enums.ItemType = enums.ItemType.Gallery): """ Get count of items in the database Args: item_type: type of item Returns: ``` { 'count': int } ``` """ item_type = enums.ItemType.get(item_type) _, db_model = item_type._msg_and_model() s = constants.db_session() return message.Identity('count', {'count': s.query(db_model).count()})
def search(cls, key, args=[], session=None): "Check if gallery contains keyword" if not session: session = constants.db_session() q = session.query(cls.id) if key: is_exclude = False if key[0] == '-' else True key = key[1:] if not is_exclude else key helper = utils._ValidContainerHelper() if not ':' in key: for g_attr in [cls.title, cls.info]: if constants.Search.Regex in args: helper.add(q.filter(g_attr.ilike(key)).all()) else: helper.add( q.filter(g_attr.ilike("%{}%".format(key))).all()) return helper.done() else: ids = q.all() return ids if ids else None
def _update_db(self, stale_cover, item_id, model, old_hash): log.d("Updating profile for database item", model) s = constants.db_session() cover = s.query(db.Profile).filter( db.and_op(db.Profile.data == old_hash, db.Profile.size == stale_cover.size)).all() if len(cover) > 1: cover, *cover_ex = cover for x in cover_ex: s.delete(x) elif cover: cover = cover[0] new = False if cover: # sometimes an identical img has already been generated and exists so we shouldnt do anything fs = io_cmd.CoreFS(cover.path) if (cover.path != stale_cover.path) and fs.exists: fs.delete() else: cover = db.Profile() new = True cover.data = stale_cover.data cover.path = stale_cover.path cover.size = stale_cover.size if new or not s.query(db.Profile).join( db.relationship_column(model, db.Profile)).filter( db.and_op(db.Profile.id == cover.id, model.id == item_id)).scalar(): log.d("Adding new profile to database item", model, "()".format(item_id)) i = s.query(model).get(item_id) i.profiles.append(cover) s.commit() self.next_progress()
def add_to_filter(gallery_id: int = 0, item_id: int = 0, item: dict = {}): """ Add a gallery to a galleryfilter Args: gallery_id: id of gallery item_id: id of existing galleryfilter, mutually exclusive with ``item`` parameter item: filter message object, mutually exclusive with ``item_id`` parameter Returns: bool whether gallery was added to filter or not """ if not gallery_id: raise exceptions.APIError(utils.this_function(), "gallery_id must be a valid gallery id") g = database_cmd.GetModelItems().run(db.Gallery, {gallery_id}) if not g: raise exceptions.DatabaseItemNotFoundError( utils.this_function(), "'{}' with id '{}' was not found".format( enums.ItemType.Gallery.name, gallery_id)) g = g[0] if item_id: p = database_cmd.GetModelItems().run(db.GalleryFilter, {item_id}) if not p: raise exceptions.DatabaseItemNotFoundError( utils.this_function(), "'{}' with id '{}' was not found".format( enums.ItemType.GalleryFilter.name, item_id)) p = p[0] elif item: p = message.GalleryFilter.from_json(item) g.filters.append(p) s = constants.db_session() s.add(g) s.commit() return message.Identity("status", True)
def main(self, model: db.Base, item_id: int, image_size: enums.ImageSize) -> db.Profile: self.model = model if image_size == enums.ImageSize.Original: image_size = utils.ImageSize(0, 0) else: image_size = utils.ImageSize( *constants.image_sizes[image_size.name.lower()]) with self.models.call() as plg: for p in plg.all(default=True): self._supported_models.update(p) if self.model not in self._supported_models: raise exceptions.CommandError( utils.this_command(self), "Model '{}' is not supported".format(model)) img_hash = io_cmd.ImageItem.gen_hash(model, image_size, item_id) cover_path = "" generate = True sess = constants.db_session() self.cover = sess.query( db.Profile).filter(db.Profile.data == img_hash).one_or_none() if self.cover: if io_cmd.CoreFS(self.cover.path).exists: generate = False else: cover_path = self.cover.path if generate: self.cover = self.run_native(self._generate_and_add, img_hash, generate, cover_path, model, item_id, image_size).get() self.cover_event.emit(self.cover) return self.cover