def data(self, load_values=False, load_collections=False, bypass_exclusions=False, propagate_bypass=False, force_value_load=None): """ Params: load_values -- Queries database for unloaded values load_collections -- Queries database to fetch all items in a collection """ self._before_data() if force_value_load is not None: self._properties['force_value_load'] = self._dict_tree_unpack( force_value_load) with db.no_autoflush(self._sess): ex = tuple(x for x, y in self._properties['exclusions'].items() if not y) gattribs = db.table_attribs( self.item, not load_values, descriptors=True, raise_err=not self._detached, exclude=ex if not bypass_exclusions else tuple(), allow=tuple(self._properties['force_value_load'].keys())) r = { x: self._unpack(x, gattribs[x], load_values, load_collections, propagate_bypass) for x in gattribs } return r
def data(self, load_values=False, load_collections=False, bypass_exclusions=False, propagate_bypass=False, force_value_load=None): """ Params: load_values -- Queries database for unloaded values load_collections -- Queries database to fetch all items in a collection """ self._before_data() if force_value_load is not None: self._properties['force_value_load'] = self._dict_tree_unpack(force_value_load) with db.no_autoflush(self._sess): ex = tuple(x for x, y in self._properties['exclusions'].items() if not y) default_ex = ('user',) gattribs = db.table_attribs(self.item, not load_values, descriptors=True, raise_err=not self._detached, exclude=ex + default_ex if not bypass_exclusions else default_ex, allow=tuple(self._properties['force_value_load'].keys())) for i in ('_properties',): gattribs.pop(i, False) r = {} for x, v in gattribs.items(): try: r[x] = self._unpack(x, v, load_values, load_collections, propagate_bypass) except DatabaseMessage.NoUnpack: pass return r
def _calculate(self, gallery_or_id, all_gallery_tags={}): assert isinstance(gallery_or_id, (str, int, db.Gallery)) sess = constants.db_session() with db.no_autoflush(sess): data = {} g_id = gallery_or_id.id if isinstance( gallery_or_id, db.Gallery) else gallery_or_id g_id = str(g_id) # because JSON keys are str tag_count = 0 tag_count_minimum = 3 if g_id in all_gallery_tags: g_tags = all_gallery_tags[g_id] for a, b in g_tags.items(): tag_count += len(b) self.set_max_progress(len(g_tags) + 3) else: if isinstance(gallery_or_id, db.Gallery): g_tags = gallery_or_id else: g_tags = database_cmd.GetModelItems().run( db.Taggable, join=db.Gallery.taggable, filter=db.Gallery.id == int(g_id)) if g_tags: g_tags = g_tags[0] tag_count = g_tags.tags.count() if tag_count > tag_count_minimum: g_tags = g_tags.compact_tags(g_tags.tags.all()) self.next_progress() if g_tags and tag_count > tag_count_minimum: log.d("Calculating similarity") g_tags = self._get_set(g_tags) data[g_id] = gl_data = {} update_dict = not all_gallery_tags max_prog = 3 for t_id, t in all_gallery_tags.items( ) or constants.db_session().query( db.Gallery.id, db.Taggable).join(db.Gallery.taggable): t_id = str(t_id) self.next_progress() if update_dict: all_gallery_tags[t_id] = t.compact_tags(t.tags.all()) max_prog += 1 self.set_max_progress(max_prog) if t_id == g_id: continue t_tags = self._get_set(all_gallery_tags[t_id]) if (math.sqrt(len(g_tags)) * math.sqrt(len(t_tags))) != 0: cos = len(g_tags & t_tags) / (math.sqrt( len(g_tags))) * math.sqrt(len(t_tags)) else: cos = 0 if cos: gl_data[t_id] = cos log.d("Finished calculating similarity") self.next_progress() return data
def _generate_gallery_fs(self, found_paths, options): paths_len = len(found_paths) galleries = [] sess = constants.db_session() with db.no_autoflush(sess): for n, p in enumerate(found_paths, 1): self.next_progress(text=f"[{n}/{paths_len}] {p}") if options.get(config.skip_existing_galleries.fullname): if db.Gallery.exists_by_path(p): continue g = io_cmd.GalleryFS(p) if g.evaluate(): g.load_all() self.set_progress(text=f"[{n}/{paths_len}] {g.path.path} .. OK") galleries.append(g) return galleries
def from_json(cls, msg, ignore_empty=True, skip_updating_existing=True, skip_descriptors=False, **kwargs): pref_title = msg.pop('preferred_title', False) obj = super().from_json(msg, ignore_empty, skip_updating_existing, skip_descriptors, **kwargs) with db.no_autoflush(db.object_session(obj) or constants.db_session()): if not skip_descriptors and pref_title: for mt in msg.get('titles', []): if utils.compare_json_dicts(mt, pref_title): break else: t = Title.from_json(pref_title, ignore_empty=ignore_empty, skip_updating_existing=skip_updating_existing, skip_descriptors=skip_descriptors, **kwargs) setattr(obj, 'preferred_title', t) if msg.get('pages'): obj.pages.reorder() if (msg.get('grouping') or msg.get('grouping_id')) and obj.grouping: obj.grouping.galleries.reorder() return obj
def _update(self, items, options): with db.safe_session() as sess: with db.no_autoflush(sess): obj_types = set() for i in items: if not i.id: raise exceptions.CoreError( utils.this_function(), "Cannot update an item without an id") if isinstance(i, io_cmd.GalleryFS): i = i.gallery obj_types.add(type(i)) i_sess = db.object_session(i) if i_sess and i_sess != sess: i_sess.expunge_all( ) # HACK: what if other sess was in use? sess.add(i) self.next_progress() sess.commit() if db.Gallery in obj_types: constants.invalidator.similar_gallery = True
def _add_to_db(self, items, options): with db.safe_session() as sess: with db.no_autoflush(sess): obj_types = set() for i in items: if isinstance(i, io_cmd.GalleryFS): i.load_all() i = i.gallery if isinstance(i, db.Gallery): if options.get(config.add_to_inbox.fullname): i.update('metatags', name=db.MetaTag.names.inbox) obj_types.add(type(i)) i_sess = db.object_session(i) if i_sess and i_sess != sess: i_sess.expunge_all( ) # HACK: what if other sess was in use? sess.add(i) self.next_progress() sess.commit() if db.Gallery in obj_types: constants.invalidator.similar_gallery = True
def update_item_tags(item_type: enums.ItemType = enums.ItemType.Gallery, item_id: int=0, tags: dict={}): """ Update tags on an item Args: item_type: possible items are :attr:`.ItemType.Gallery`, :attr:`.ItemType.Page`, :attr:`.ItemType.Grouping`, :attr:`.ItemType.Collection` item_id: id of item to update tags for tags: tags Returns: bool whether tags were updated or not """ if not item_id: raise exceptions.APIError(utils.this_function(), "item_id must be a valid item id") item_type = enums.ItemType.get(item_type) _, db_item = item_type._msg_and_model( (enums.ItemType.Gallery, enums.ItemType.Collection, enums.ItemType.Grouping, enums.ItemType.Page)) db_obj = database_cmd.GetModelItems().run(db_item, {item_id}) if not db_obj: raise exceptions.DatabaseItemNotFoundError(utils.this_function(), "'{}' with id '{}' was not found".format(item_type.name, item_id)) db_obj = db_obj[0] tags = _decontruct_tags_msg(tags) s = constants.db_session() with db.no_autoflush(s): s.add(db_obj) db_obj.tags = tags s.commit() return message.Identity("status", True)
def from_json(cls, msg, ignore_empty=True, skip_updating_existing=True, skip_descriptors=False, _type=None): db_obj = None with db.no_autoflush(constants.db_session()) as sess: if not cls.db_type and _type is None: raise ValueError("A database type has not been set") db_type = _type or cls.db_type item_attrs = db.table_attribs(db_type, id=None, descriptors=True) if msg and not all((k in item_attrs.keys() for k in msg.keys())): m_keys = set(msg.keys()).difference(set(item_attrs.keys())) raise exceptions.InvalidMessage( utils.this_function(), f"Message object mismatch. Message contains keys that are not present in the corresponding database item: {m_keys}" ) if msg: obj_id = msg.get('id', False) if obj_id: db_obj = sess.query(db_type).get(obj_id) else: db_obj = db_type() if not (obj_id and db_obj and skip_updating_existing): for attr, value in msg.items(): if ignore_empty: if value is None: continue elif isinstance(value, (list, dict)) and not value: continue cls_attr = item_attrs[attr] if skip_descriptors and db.is_descriptor(cls_attr): continue obj_attr = getattr(db_obj, attr) try: col_model = db.column_model(cls_attr) except TypeError: # most likely a hybrid_property descriptor col_model = None if not issubclass(col_model, db.Base) if inspect.isclass( col_model) else not isinstance( col_model, db.Base): if isinstance( col_model, (db.Boolean, db.Integer, db.Password, db.String, db.Text, db.LowerCaseString, db.CapitalizedString)): setattr(db_obj, attr, value) elif isinstance(col_model, db.ArrowType): setattr(db_obj, attr, arrow.Arrow.fromtimestamp(value)) elif db.is_descriptor(cls_attr): if db.descriptor_has_setter(cls_attr): raise NotImplementedError else: raise NotImplementedError( f"Value deserializing for this database type does not exist ({col_model})" ) continue msg_obj = cls._get_message_object( cls, attr, col_model, check_recursive=False, class_type=True) if issubclass(col_model, db.MetaTag): setattr(db_obj, attr, msg_obj._pack_metatags(value)) elif db.is_list(cls_attr) or db.is_query(cls_attr): for v in value: obj_attr.append( msg_obj.from_json( v, _type=col_model, ignore_empty=ignore_empty, skip_updating_existing= skip_updating_existing, skip_descriptors=skip_descriptors)) elif db.is_descriptor(cls_attr): if db.descriptor_has_setter(cls_attr): raise NotImplementedError else: setattr( db_obj, attr, msg_obj.from_json( value, _type=col_model, ignore_empty=ignore_empty, skip_updating_existing= skip_updating_existing, skip_descriptors=skip_descriptors)) return db_obj
def from_json(cls, msg, ignore_empty=True, skip_updating_existing=True, skip_descriptors=False, **kwargs): obj = super().from_json(msg, ignore_empty, skip_updating_existing, skip_descriptors, **kwargs) with db.no_autoflush(db.object_session(obj) or constants.db_session()): if msg.get('galleries'): obj.galleries.reorder() return obj
def from_json(cls, msg, ignore_empty=True, skip_updating_existing=True, skip_descriptors=False, _type=None, ignore_private=True, _from_attr=''): db_obj = None if not msg: return db_obj if not isinstance(msg, dict): raise exceptions.InvalidMessage( utils.this_function(), f"Expected message object on '{_from_attr}' not '{type(msg).__name__}'") with db.no_autoflush(constants.db_session()) as sess: if not cls.db_type and _type is None: raise ValueError("A database type has not been set") db_type = _type or cls.db_type item_attrs = db.table_attribs(db_type, id=None, descriptors=True) if msg and not all((k in item_attrs.keys() for k in msg.keys())): m_keys = set(msg.keys()).difference(set(item_attrs.keys())) raise exceptions.InvalidMessage( utils.this_function(), f"Message object mismatch. '{_from_attr}' contains keys that are not present in the corresponding database item: {m_keys}") if msg: new_obj = True obj_id = msg.get('id', False) if obj_id: db_obj = sess.query(db_type).get(obj_id) if not db_obj: raise exceptions.CoreError(utils.this_function(), f"Existing item from message object with id '{obj_id}' was not found") new_obj = False else: db_obj = db_type() if new_obj and isinstance(db_obj, db.UniqueMixin): if isinstance(db_obj, db.NameMixin) and msg.get('name') is not None: db_obj = db_type.as_unique(name=msg.get('name')) new_obj = bool(db_obj.id) elif isinstance(db_obj, db.NamespaceTags): ns_name = None tag_name = None try: ns_name = utils.get_dict_value('namespace.name', msg) tag_name = utils.get_dict_value('tag.name', msg) except KeyError: pass if ns_name is not None and tag_name is not None: db_obj = db_type.as_unique(ns=ns_name, tag=tag_name) new_obj = bool(db_obj.id) elif isinstance(db_obj, (db.Artist, db.Parody)): a_names = set() if msg.get('preferred_name') and msg['preferred_name'].get('name') is not None: a_names.add(msg['preferred_name']['name']) if msg.get('names'): for n in msg['names']: if n.get('name') is not None: a_names.add(n['name']) if a_names: db_obj = db_type.as_unique(names=a_names) new_obj = bool(db_obj.id) if db_obj and not (obj_id and db_obj and skip_updating_existing): for attr, value in msg.items(): if attr == 'id': continue if ignore_private and attr.startswith('_'): continue if ignore_empty: if value is None: continue elif isinstance(value, (list, dict)) and not value: continue cls_attr = item_attrs[attr] if skip_descriptors and db.is_descriptor(cls_attr): continue obj_attr = getattr(db_obj, attr) # noqa: F841 try: col_model = db.column_model(cls_attr) except TypeError: # most likely a hybrid_property descriptor col_model = None if not issubclass(col_model, db.Base) if inspect.isclass( col_model) else not isinstance(col_model, db.Base): if isinstance(col_model, (db.Boolean, db.Integer, db.Password, db.String, db.Text, db.LowerCaseString, db.CapitalizedString)): setattr(db_obj, attr, value) elif isinstance(col_model, db.ArrowType): setattr(db_obj, attr, arrow.Arrow.fromtimestamp(value)) elif db.is_descriptor(cls_attr): if db.descriptor_has_setter(cls_attr): raise NotImplementedError else: raise NotImplementedError( f"Value deserializing for this database type does not exist ({col_model})") continue msg_obj = cls._get_message_object(cls, attr, col_model, check_recursive=False, class_type=True) if col_model == db.Taggable and isinstance(value, dict): if db_obj.taggable and db_obj.taggable.id: value['id'] = db_obj.taggable.id if issubclass(col_model, db.MetaTag): setattr(db_obj, attr, msg_obj._pack_metatags(value)) elif db.is_list(cls_attr) or db.is_query(cls_attr): n_l = [] for v in value: o = msg_obj.from_json( v, _type=col_model, ignore_empty=ignore_empty, skip_updating_existing=skip_updating_existing, skip_descriptors=skip_descriptors, _from_attr=_from_attr + '.' + attr if _from_attr else attr, ) if o is not None: n_l.append(o) setattr(db_obj, attr, n_l) elif db.is_descriptor(cls_attr): if db.descriptor_has_setter(cls_attr): raise NotImplementedError else: setattr( db_obj, attr, msg_obj.from_json( value, _type=col_model, _from_attr=_from_attr + '.' + attr if _from_attr else attr, ignore_empty=ignore_empty, skip_updating_existing=skip_updating_existing, skip_descriptors=skip_descriptors)) return db_obj