def data(self,
load_values=False,
load_collections=False,
bypass_exclusions=False,
propagate_bypass=False,
force_value_load=None):
"""
Params:
load_values -- Queries database for unloaded values
load_collections -- Queries database to fetch all items in a collection
"""
self._before_data()
if force_value_load is not None:
self._properties['force_value_load'] = self._dict_tree_unpack(
force_value_load)
with db.no_autoflush(self._sess):
ex = tuple(x for x, y in self._properties['exclusions'].items()
if not y)
gattribs = db.table_attribs(
self.item,
not load_values,
descriptors=True,
raise_err=not self._detached,
exclude=ex if not bypass_exclusions else tuple(),
allow=tuple(self._properties['force_value_load'].keys()))
r = {
x: self._unpack(x, gattribs[x], load_values, load_collections,
propagate_bypass)
for x in gattribs
}
return r
def data(self, load_values=False, load_collections=False,
bypass_exclusions=False, propagate_bypass=False,
force_value_load=None):
"""
Params:
load_values -- Queries database for unloaded values
load_collections -- Queries database to fetch all items in a collection
"""
self._before_data()
if force_value_load is not None:
self._properties['force_value_load'] = self._dict_tree_unpack(force_value_load)
with db.no_autoflush(self._sess):
ex = tuple(x for x, y in self._properties['exclusions'].items() if not y)
default_ex = ('user',)
gattribs = db.table_attribs(self.item, not load_values, descriptors=True, raise_err=not self._detached,
exclude=ex + default_ex if not bypass_exclusions else default_ex,
allow=tuple(self._properties['force_value_load'].keys()))
for i in ('_properties',):
gattribs.pop(i, False)
r = {}
for x, v in gattribs.items():
try:
r[x] = self._unpack(x, v, load_values, load_collections, propagate_bypass)
except DatabaseMessage.NoUnpack:
pass
return r
def _calculate(self, gallery_or_id, all_gallery_tags={}):
assert isinstance(gallery_or_id, (str, int, db.Gallery))
sess = constants.db_session()
with db.no_autoflush(sess):
data = {}
g_id = gallery_or_id.id if isinstance(
gallery_or_id, db.Gallery) else gallery_or_id
g_id = str(g_id) # because JSON keys are str
tag_count = 0
tag_count_minimum = 3
if g_id in all_gallery_tags:
g_tags = all_gallery_tags[g_id]
for a, b in g_tags.items():
tag_count += len(b)
self.set_max_progress(len(g_tags) + 3)
else:
if isinstance(gallery_or_id, db.Gallery):
g_tags = gallery_or_id
else:
g_tags = database_cmd.GetModelItems().run(
db.Taggable,
join=db.Gallery.taggable,
filter=db.Gallery.id == int(g_id))
if g_tags:
g_tags = g_tags[0]
tag_count = g_tags.tags.count()
if tag_count > tag_count_minimum:
g_tags = g_tags.compact_tags(g_tags.tags.all())
self.next_progress()
if g_tags and tag_count > tag_count_minimum:
log.d("Calculating similarity")
g_tags = self._get_set(g_tags)
data[g_id] = gl_data = {}
update_dict = not all_gallery_tags
max_prog = 3
for t_id, t in all_gallery_tags.items(
) or constants.db_session().query(
db.Gallery.id, db.Taggable).join(db.Gallery.taggable):
t_id = str(t_id)
self.next_progress()
if update_dict:
all_gallery_tags[t_id] = t.compact_tags(t.tags.all())
max_prog += 1
self.set_max_progress(max_prog)
if t_id == g_id:
continue
t_tags = self._get_set(all_gallery_tags[t_id])
if (math.sqrt(len(g_tags)) * math.sqrt(len(t_tags))) != 0:
cos = len(g_tags & t_tags) / (math.sqrt(
len(g_tags))) * math.sqrt(len(t_tags))
else:
cos = 0
if cos:
gl_data[t_id] = cos
log.d("Finished calculating similarity")
self.next_progress()
return data
def _generate_gallery_fs(self, found_paths, options):
paths_len = len(found_paths)
galleries = []
sess = constants.db_session()
with db.no_autoflush(sess):
for n, p in enumerate(found_paths, 1):
self.next_progress(text=f"[{n}/{paths_len}] {p}")
if options.get(config.skip_existing_galleries.fullname):
if db.Gallery.exists_by_path(p):
continue
g = io_cmd.GalleryFS(p)
if g.evaluate():
g.load_all()
self.set_progress(text=f"[{n}/{paths_len}] {g.path.path} .. OK")
galleries.append(g)
return galleries
def from_json(cls, msg, ignore_empty=True, skip_updating_existing=True, skip_descriptors=False, **kwargs):
pref_title = msg.pop('preferred_title', False)
obj = super().from_json(msg, ignore_empty, skip_updating_existing, skip_descriptors, **kwargs)
with db.no_autoflush(db.object_session(obj) or constants.db_session()):
if not skip_descriptors and pref_title:
for mt in msg.get('titles', []):
if utils.compare_json_dicts(mt, pref_title):
break
else:
t = Title.from_json(pref_title, ignore_empty=ignore_empty, skip_updating_existing=skip_updating_existing,
skip_descriptors=skip_descriptors, **kwargs)
setattr(obj, 'preferred_title', t)
if msg.get('pages'):
obj.pages.reorder()
if (msg.get('grouping') or msg.get('grouping_id')) and obj.grouping:
obj.grouping.galleries.reorder()
return obj
def _update(self, items, options):
with db.safe_session() as sess:
with db.no_autoflush(sess):
obj_types = set()
for i in items:
if not i.id:
raise exceptions.CoreError(
utils.this_function(),
"Cannot update an item without an id")
if isinstance(i, io_cmd.GalleryFS):
i = i.gallery
obj_types.add(type(i))
i_sess = db.object_session(i)
if i_sess and i_sess != sess:
i_sess.expunge_all(
) # HACK: what if other sess was in use?
sess.add(i)
self.next_progress()
sess.commit()
if db.Gallery in obj_types:
constants.invalidator.similar_gallery = True
def _add_to_db(self, items, options):
with db.safe_session() as sess:
with db.no_autoflush(sess):
obj_types = set()
for i in items:
if isinstance(i, io_cmd.GalleryFS):
i.load_all()
i = i.gallery
if isinstance(i, db.Gallery):
if options.get(config.add_to_inbox.fullname):
i.update('metatags', name=db.MetaTag.names.inbox)
obj_types.add(type(i))
i_sess = db.object_session(i)
if i_sess and i_sess != sess:
i_sess.expunge_all(
) # HACK: what if other sess was in use?
sess.add(i)
self.next_progress()
sess.commit()
if db.Gallery in obj_types:
constants.invalidator.similar_gallery = True
def update_item_tags(item_type: enums.ItemType = enums.ItemType.Gallery,
item_id: int=0,
tags: dict={}):
"""
Update tags on an item
Args:
item_type: possible items are :attr:`.ItemType.Gallery`, :attr:`.ItemType.Page`,
:attr:`.ItemType.Grouping`, :attr:`.ItemType.Collection`
item_id: id of item to update tags for
tags: tags
Returns:
bool whether tags were updated or not
"""
if not item_id:
raise exceptions.APIError(utils.this_function(), "item_id must be a valid item id")
item_type = enums.ItemType.get(item_type)
_, db_item = item_type._msg_and_model(
(enums.ItemType.Gallery, enums.ItemType.Collection, enums.ItemType.Grouping, enums.ItemType.Page))
db_obj = database_cmd.GetModelItems().run(db_item, {item_id})
if not db_obj:
raise exceptions.DatabaseItemNotFoundError(utils.this_function(),
"'{}' with id '{}' was not found".format(item_type.name,
item_id))
db_obj = db_obj[0]
tags = _decontruct_tags_msg(tags)
s = constants.db_session()
with db.no_autoflush(s):
s.add(db_obj)
db_obj.tags = tags
s.commit()
return message.Identity("status", True)
def from_json(cls,
msg,
ignore_empty=True,
skip_updating_existing=True,
skip_descriptors=False,
_type=None):
db_obj = None
with db.no_autoflush(constants.db_session()) as sess:
if not cls.db_type and _type is None:
raise ValueError("A database type has not been set")
db_type = _type or cls.db_type
item_attrs = db.table_attribs(db_type, id=None, descriptors=True)
if msg and not all((k in item_attrs.keys() for k in msg.keys())):
m_keys = set(msg.keys()).difference(set(item_attrs.keys()))
raise exceptions.InvalidMessage(
utils.this_function(),
f"Message object mismatch. Message contains keys that are not present in the corresponding database item: {m_keys}"
)
if msg:
obj_id = msg.get('id', False)
if obj_id:
db_obj = sess.query(db_type).get(obj_id)
else:
db_obj = db_type()
if not (obj_id and db_obj and skip_updating_existing):
for attr, value in msg.items():
if ignore_empty:
if value is None:
continue
elif isinstance(value, (list, dict)) and not value:
continue
cls_attr = item_attrs[attr]
if skip_descriptors and db.is_descriptor(cls_attr):
continue
obj_attr = getattr(db_obj, attr)
try:
col_model = db.column_model(cls_attr)
except TypeError: # most likely a hybrid_property descriptor
col_model = None
if not issubclass(col_model,
db.Base) if inspect.isclass(
col_model) else not isinstance(
col_model, db.Base):
if isinstance(
col_model,
(db.Boolean, db.Integer, db.Password,
db.String, db.Text, db.LowerCaseString,
db.CapitalizedString)):
setattr(db_obj, attr, value)
elif isinstance(col_model, db.ArrowType):
setattr(db_obj, attr,
arrow.Arrow.fromtimestamp(value))
elif db.is_descriptor(cls_attr):
if db.descriptor_has_setter(cls_attr):
raise NotImplementedError
else:
raise NotImplementedError(
f"Value deserializing for this database type does not exist ({col_model})"
)
continue
msg_obj = cls._get_message_object(
cls,
attr,
col_model,
check_recursive=False,
class_type=True)
if issubclass(col_model, db.MetaTag):
setattr(db_obj, attr,
msg_obj._pack_metatags(value))
elif db.is_list(cls_attr) or db.is_query(cls_attr):
for v in value:
obj_attr.append(
msg_obj.from_json(
v,
_type=col_model,
ignore_empty=ignore_empty,
skip_updating_existing=
skip_updating_existing,
skip_descriptors=skip_descriptors))
elif db.is_descriptor(cls_attr):
if db.descriptor_has_setter(cls_attr):
raise NotImplementedError
else:
setattr(
db_obj, attr,
msg_obj.from_json(
value,
_type=col_model,
ignore_empty=ignore_empty,
skip_updating_existing=
skip_updating_existing,
skip_descriptors=skip_descriptors))
return db_obj
def from_json(cls, msg, ignore_empty=True, skip_updating_existing=True, skip_descriptors=False, **kwargs):
obj = super().from_json(msg, ignore_empty, skip_updating_existing, skip_descriptors, **kwargs)
with db.no_autoflush(db.object_session(obj) or constants.db_session()):
if msg.get('galleries'):
obj.galleries.reorder()
return obj
def from_json(cls, msg, ignore_empty=True, skip_updating_existing=True,
skip_descriptors=False, _type=None,
ignore_private=True, _from_attr=''):
db_obj = None
if not msg:
return db_obj
if not isinstance(msg, dict):
raise exceptions.InvalidMessage(
utils.this_function(),
f"Expected message object on '{_from_attr}' not '{type(msg).__name__}'")
with db.no_autoflush(constants.db_session()) as sess:
if not cls.db_type and _type is None:
raise ValueError("A database type has not been set")
db_type = _type or cls.db_type
item_attrs = db.table_attribs(db_type, id=None, descriptors=True)
if msg and not all((k in item_attrs.keys() for k in msg.keys())):
m_keys = set(msg.keys()).difference(set(item_attrs.keys()))
raise exceptions.InvalidMessage(
utils.this_function(),
f"Message object mismatch. '{_from_attr}' contains keys that are not present in the corresponding database item: {m_keys}")
if msg:
new_obj = True
obj_id = msg.get('id', False)
if obj_id:
db_obj = sess.query(db_type).get(obj_id)
if not db_obj:
raise exceptions.CoreError(utils.this_function(),
f"Existing item from message object with id '{obj_id}' was not found")
new_obj = False
else:
db_obj = db_type()
if new_obj and isinstance(db_obj, db.UniqueMixin):
if isinstance(db_obj, db.NameMixin) and msg.get('name') is not None:
db_obj = db_type.as_unique(name=msg.get('name'))
new_obj = bool(db_obj.id)
elif isinstance(db_obj, db.NamespaceTags):
ns_name = None
tag_name = None
try:
ns_name = utils.get_dict_value('namespace.name', msg)
tag_name = utils.get_dict_value('tag.name', msg)
except KeyError:
pass
if ns_name is not None and tag_name is not None:
db_obj = db_type.as_unique(ns=ns_name, tag=tag_name)
new_obj = bool(db_obj.id)
elif isinstance(db_obj, (db.Artist, db.Parody)):
a_names = set()
if msg.get('preferred_name') and msg['preferred_name'].get('name') is not None:
a_names.add(msg['preferred_name']['name'])
if msg.get('names'):
for n in msg['names']:
if n.get('name') is not None:
a_names.add(n['name'])
if a_names:
db_obj = db_type.as_unique(names=a_names)
new_obj = bool(db_obj.id)
if db_obj and not (obj_id and db_obj and skip_updating_existing):
for attr, value in msg.items():
if attr == 'id':
continue
if ignore_private and attr.startswith('_'):
continue
if ignore_empty:
if value is None:
continue
elif isinstance(value, (list, dict)) and not value:
continue
cls_attr = item_attrs[attr]
if skip_descriptors and db.is_descriptor(cls_attr):
continue
obj_attr = getattr(db_obj, attr) # noqa: F841
try:
col_model = db.column_model(cls_attr)
except TypeError: # most likely a hybrid_property descriptor
col_model = None
if not issubclass(col_model, db.Base) if inspect.isclass(
col_model) else not isinstance(col_model, db.Base):
if isinstance(col_model, (db.Boolean, db.Integer, db.Password,
db.String, db.Text, db.LowerCaseString,
db.CapitalizedString)):
setattr(db_obj, attr, value)
elif isinstance(col_model, db.ArrowType):
setattr(db_obj, attr, arrow.Arrow.fromtimestamp(value))
elif db.is_descriptor(cls_attr):
if db.descriptor_has_setter(cls_attr):
raise NotImplementedError
else:
raise NotImplementedError(
f"Value deserializing for this database type does not exist ({col_model})")
continue
msg_obj = cls._get_message_object(cls, attr, col_model, check_recursive=False, class_type=True)
if col_model == db.Taggable and isinstance(value, dict):
if db_obj.taggable and db_obj.taggable.id:
value['id'] = db_obj.taggable.id
if issubclass(col_model, db.MetaTag):
setattr(db_obj, attr, msg_obj._pack_metatags(value))
elif db.is_list(cls_attr) or db.is_query(cls_attr):
n_l = []
for v in value:
o = msg_obj.from_json(
v,
_type=col_model,
ignore_empty=ignore_empty,
skip_updating_existing=skip_updating_existing,
skip_descriptors=skip_descriptors,
_from_attr=_from_attr + '.' + attr if _from_attr else attr,
)
if o is not None:
n_l.append(o)
setattr(db_obj, attr, n_l)
elif db.is_descriptor(cls_attr):
if db.descriptor_has_setter(cls_attr):
raise NotImplementedError
else:
setattr(
db_obj,
attr,
msg_obj.from_json(
value,
_type=col_model,
_from_attr=_from_attr + '.' + attr if _from_attr else attr,
ignore_empty=ignore_empty,
skip_updating_existing=skip_updating_existing,
skip_descriptors=skip_descriptors))
return db_obj