def migrate_rules(ruletypes=["read", "write", "data"]): """ WARNING: This function creates db objects that may trigger unwanted permission update functions on session flushing. Disable database triggers (see `disabled_triggers`) in migration scripts when using this function! """ for ruletype in ruletypes: logg.info("------ migrating %s permissions ------", ruletype) nid_to_rulesets, nid_to_special_rulestrings = load_node_rules( ruletype + "access") nid_to_special_rulestrings = { nid: ",".join(r for r in rulestrings if r is not None) for nid, rulestrings in iteritems(nid_to_special_rulestrings) } nid_to_special_rules = convert_nid_to_rulestr( nid_to_special_rulestrings) save_node_to_ruleset_mappings(nid_to_rulesets, ruletype) db.session.flush() save_node_to_special_rules( {k: v for k, v in iteritems(nid_to_special_rules) if v}, ruletype) db.session.flush() create_rulemappings_stmt = sql.select( [mediatumfunc.create_node_rulemappings_from_rulesets(ruletype)]) db.session.execute(create_rulemappings_stmt)
def _extract_metadata(self, files=None): image_file = self._find_processing_file(files) width, height = get_image_dimensions(image_file) # XXX: this is a bit redundant... self.set("origwidth", width) self.set("origheight", height) self.set("origsize", image_file.size) self.set("width", width) self.set("height", height) # Exif unwanted_attrs = Image.get_unwanted_exif_attributes() with open(image_file.abspath, 'rb') as f: tags = EXIF.process_file(f) for k in tags.keys(): # don't set unwanted exif attributes if any(tag in k for tag in unwanted_attrs): continue if tags[k]: self.set("exif_" + k.replace(" ", "_"), utf8_decode_escape(str(tags[k]))) # IPTC iptc_metadata = lib.iptc.IPTC.get_iptc_tags(image_file.abspath) if iptc_metadata is not None: for k, v in iteritems(iptc_metadata): self.set('iptc_' + k, v)
def position_filter(sortfields_to_comp, after=False, before=False): assert before or after iter_sortfield = iteritems(sortfields_to_comp) prev_sortfields_to_comp = [] first_sortfield = next(iter_sortfield) cond = make_cond(first_sortfield, invert=before) prev_sortfields_to_comp.append(first_sortfield) # add remaining sortfields if present for sortfield_to_comp in iter_sortfield: # We are doing a "tuple" comparision here. If the first field is equivalent, we have to compare the seconds field and so on. # first add comparison for current field... sub_cond = make_cond(sortfield_to_comp, invert=before) # ... and add equivalence conditions for previous fields for prev in prev_sortfields_to_comp: expr = node_value_expression(prev[0]) # XXX: can we replace this by make_cond or something like that? eq_cond = expr == _prepare_value(prev[0], prev[1][1]) sub_cond &= eq_cond prev_sortfields_to_comp.append(sortfield_to_comp) # or-append condition for current field to full condition if cond is None: cond = sub_cond else: cond |= sub_cond return cond
def add_node_to_xmldoc( node, xmlroot, written=set(), children=True, exclude_filetypes=[], exclude_childtypes=[], attribute_name_filter=None): from schema.schema import Mask from schema.mapping import Mapping written.add(node.id) xmlnode = etree.SubElement(xmlroot, "node") xmlnode.set("name", node.name or u"") xmlnode.set("id", unicode(node.id)) xmlnode.set("type", (node.type + "/" + (node.schema or u"")).strip("/")) xmlnode.set("datatype", node.type) xmlnode.set("schema", (node.schema or u"")) # TODO: no access rights at the moment for name, value in sorted(iteritems(node.attrs)): if attribute_name_filter and not attribute_name_filter(name): continue xmlattr = etree.SubElement(xmlnode, "attribute") xmlattr.set("name", name) # protect XML from invalid characters # XXX: is this ok? xmlattr.text = etree.CDATA(xml_remove_illegal_chars(unicode(value))) files = [f for f in node.file_objects if f.filetype != u"metadata"] if exclude_filetypes: files = [f for f in files if f.filetype not in (exclude_filetypes)] for fileobj in files: add_file_to_xmlnode(fileobj, xmlnode) if children: child_query = node.children if exclude_childtypes: child_query = child_query.filter(~Node.type.in_(exclude_childtypes)) for child in child_query.order_by("orderpos"): add_child_to_xmlnode(child, xmlnode) if child.id not in written: add_node_to_xmldoc(child, xmlroot, written, children, exclude_filetypes, exclude_childtypes, attribute_name_filter) if isinstance(node, Mask): exportmapping_id = node.get(u"exportmapping").strip() if exportmapping_id and exportmapping_id not in written: mapping = q(Mapping).get(int(exportmapping_id)) if mapping is not None: written.add(mapping.id) add_node_to_xmldoc(mapping, xmlroot, written, children, exclude_filetypes, exclude_childtypes, attribute_name_filter) return xmlnode
def get_maskcache_report(maskcache_accesscount): sorted_entries = [(k, v) for k, v in sorted(iteritems(maskcache_accesscount))] total_access_count = sum(v for k, v in sorted_entries) num_cache_keys = len(sorted_entries) return "keys: %s, total access count: %s, %s" % ( num_cache_keys, total_access_count, sorted_entries)
def get_user_data(self, data): return { attribute_name: data.get(ldap_fieldname)[0].decode("utf8") if data.get(ldap_fieldname) else None for attribute_name, ldap_fieldname in iteritems( self.user_attributes) }
def save_node_to_special_rules(nid_to_special_rules, ruletype): from core import Node logg.info("saving special %s rules for %d nodes", ruletype, len(nid_to_special_rules)) for nid, rules_with_flags in iteritems(nid_to_special_rules): ruleset_assoc = _create_private_ruleset_assoc_for_nid(nid, ruletype) db.session.add(ruleset_assoc) rs = ruleset_assoc.ruleset assert rules_with_flags, "tried to save a special ruleset with no rules" for rule, (invert, blocking) in rules_with_flags: rs.rule_assocs.append(AccessRulesetToRule(rule=rule, invert=invert, blocking=blocking))
def find_plugin_with_theme(theme_name): """Look for a plugin which contains the requested theme `theme_name and return its path.` :param theme_name: name of the theme (name of theme directory) :returns: path to plugin """ for plugin_name, m in iteritems(plugins): plugin_path = os.path.dirname(m.__file__) logg.debug("looking for theme %s in plugin %s", theme_name, plugin_name) theme_path = os.path.join(plugin_path, "themes", theme_name) if os.path.exists(theme_path): return plugin_path
def print_url(self): if config.getboolean("config.enable_printing"): # self.content means: we're showing a single result node. # Therefore, we want to print the node, not the list. if self.content is not None: return self.content.print_url if self.container.system_attrs.get("print", "1") == "1": # printing is allowed for containers by default, unless system.print != "1" is set on the node params = {k:v for k, v in iteritems(self.nav_params) if k.startswith("sortfield")} return print_url(self.container.id, **params)
def search_link(self, mode="simple"): params = { k: v for k, v in iteritems(self.url_params) if k not in ("query", "searchmode", "id") } if mode != "simple": params["searchmode"] = mode return node_url(self.container.id, ** params) if not self.edit else edit_node_url( self.container.id, **params)
def nav_link(self, **param_overrides): params = {} # params = self.nav_params.copy() params = self.nav_searchparams.copy() params["page"] = self.page if not "page" in param_overrides: if self.page: params["page"] = self.page params.update(param_overrides) params = {k: v.encode('utf-8') if not isinstance(v, int) else v for k, v in iteritems(params) if v is not None and v != ""} return '/edit/edit_content?' + urllib.urlencode(params)
def assert_access_rule(access_rule, invert=None, blocking=None, **attrs_to_check): # we must flush the rule first to gain access to column default values from core import db db.session.add(access_rule) db.session.flush() attrs = dict(group_ids=None, dateranges=None, subnets=None, invert_group=False, invert_date=False, invert_subnet=False) attrs.update(attrs_to_check) for attrname, expected_value in iteritems(attrs): value = getattr(access_rule, attrname) if isinstance(value, set): assert value == set(expected_value) else: assert value == expected_value
def save_node_to_special_rules(nid_to_special_rules, ruletype): from core import Node logg.info("saving special %s rules for %d nodes", ruletype, len(nid_to_special_rules)) for nid, rules_with_flags in iteritems(nid_to_special_rules): ruleset_assoc = _create_private_ruleset_assoc_for_nid(nid, ruletype) db.session.add(ruleset_assoc) rs = ruleset_assoc.ruleset assert rules_with_flags, "tried to save a special ruleset with no rules" for rule, (invert, blocking) in rules_with_flags: rs.rule_assocs.append( AccessRulesetToRule(rule=rule, invert=invert, blocking=blocking))
def all_node_id_gen(api, start_node_id=COLLECTIONS_NODE_ID): """Generates all node ids in the tree starting from `start_node_id`""" top_collections = iteritems(api.children_shortlist(start_node_id)) it = None while True: if not it: collection_nid, collection_name, _ = next(top_collections) logg.info("fetching all children of top collection %s (%s)", collection_name, collection_nid) it = api.allchildren_shortlist(collection_nid).iterids() try: nid = next(it) except StopIteration: it = None yield nid
def check_node(node, expected, **check_attrs): assert isinstance(node, Node) print("expected node content:") pprint(expected) print("actual node attributes:") pprint(node.attributes) # every doi-imported node must fulfill this assert node.name == expected[u"DOI"] doi = node.get(u"doi") or node.get(u"DOI") assert doi == expected[u"DOI"] # additional attribute value checks for attr, expected in iteritems(check_attrs): value = node.get(attr) if value: assert value == expected
def feedback(self, req): self.lang = lang(req) self.ip = req.ip self.url_params = {k: v for k, v in iteritems(req.args) if k not in ()} searchmode_from_req = req.args.get("searchmode") if searchmode_from_req in ("extended", "extendedsuper"): if self.searchmask is None: raise ValueError( "no searchmask defined, extended search cannot be run for container {}" .format(self.container.id)) self.searchmode = searchmode_from_req if self.searchmode == "extended": extendedfields = range(1, 4) elif self.searchmode == "extendedsuper": extendedfields = range(1, 11) else: extendedfields = [] # this is the "special" value for simple search if not extendedfields and "query" in req.args: self.values[0] = req.args["query"] else: for pos in extendedfields: searchmaskitem_argname = "field" + str(pos) searchmaskitem_id = req.args.get(searchmaskitem_argname, type=int) self.searchmaskitem_ids[pos] = searchmaskitem_id searchmaskitem = self.searchmask.children.filter_by( id=searchmaskitem_id).scalar( ) if searchmaskitem_id else None field = searchmaskitem.children.first( ) if searchmaskitem else None value_argname = "query" + str(pos) if field is not None and field.getFieldtype() == "date": from_value = req.args.get(value_argname + "-from", u"") to_value = req.args.get(value_argname + "-to", u"") value = from_value + ";" + to_value else: value = req.args.get(value_argname) if value: self.values[pos] = value
def migrate_rules(ruletypes=["read", "write", "data"]): """ WARNING: This function creates db objects that may trigger unwanted permission update functions on session flushing. Disable database triggers (see `disabled_triggers`) in migration scripts when using this function! """ for ruletype in ruletypes: logg.info("------ migrating %s permissions ------", ruletype) nid_to_rulesets, nid_to_special_rulestrings = load_node_rules(ruletype + "access") nid_to_special_rulestrings = {nid: ",".join(r for r in rulestrings if r is not None) for nid, rulestrings in iteritems(nid_to_special_rulestrings)} nid_to_special_rules = convert_nid_to_rulestr(nid_to_special_rulestrings) save_node_to_ruleset_mappings(nid_to_rulesets, ruletype) db.session.flush() save_node_to_special_rules({k: v for k, v in iteritems(nid_to_special_rules) if v}, ruletype) db.session.flush() create_rulemappings_stmt = sql.select([mediatumfunc.create_node_rulemappings_from_rulesets(ruletype)]) db.session.execute(create_rulemappings_stmt)
def nav_link(self, **param_overrides): """ params can be removed from the URL by setting them to None in param_overrides """ params = self.nav_params.copy() if self.liststyle_name: params["liststyle"] = self.liststyle_name if self.nodes_per_page_from_req: params["nodes_per_page"] = self.nodes_per_page_from_req if not ("before" in param_overrides or "after" in param_overrides): if self.before: params["before"] = self.before if self.after: params["after"] = self.after params.update(param_overrides) params = {k: v for k, v in iteritems(params) if v is not None} return node_url(**params)
def feedback(self, req): self.lang = lang(req) self.ip = req.ip self.url_params = {k: v for k, v in iteritems(req.args) if k not in ()} searchmode_from_req = req.args.get("searchmode") if searchmode_from_req in ("extended", "extendedsuper"): if self.searchmask is None: raise ValueError("no searchmask defined, extended search cannot be run for container {}".format(self.container.id)) self.searchmode = searchmode_from_req if self.searchmode == "extended": extendedfields = range(1, 4) elif self.searchmode == "extendedsuper": extendedfields = range(1, 11) else: extendedfields = [] # this is the "special" value for simple search if not extendedfields and "query" in req.args: self.values[0] = req.args["query"] else: for pos in extendedfields: searchmaskitem_argname = "field" + str(pos) searchmaskitem_id = req.args.get(searchmaskitem_argname, type=int) self.searchmaskitem_ids[pos] = searchmaskitem_id searchmaskitem = self.searchmask.children.filter_by(id=searchmaskitem_id).scalar() if searchmaskitem_id else None field = searchmaskitem.children.first() if searchmaskitem else None value_argname = "query" + str(pos) if field is not None and field.getFieldtype() == "date": from_value = req.args.get(value_argname + "-from", u"") to_value = req.args.get(value_argname + "-to", u"") value = from_value + ";" + to_value else: value = req.args.get(value_argname) if value: self.values[pos] = value
def view(req, op): maskcache_info = { "count": len(data.maskcache), "access_count": sorted(iteritems(data.maskcache_accesscount), key=lambda t: t[1], reverse=True), "total_size": 0 } all_objects = gc.get_objects() memory_info = {"count": len(all_objects), "total_size": 0, "summary": ""} sessions = athana._ATHANA_HANDLER.sessions sessions_info = {"count": len(sessions), "total_size": 0, "summary": ""} if pympler: maskcache_info["total_size"] = asizeof(data.maskcache) sessions_info["total_size"] = asizeof(sessions) summarized_all_objects = sorted(summary.summarize(all_objects), key=lambda t: t[2], reverse=True) memory_info["summary"] = summarized_all_objects[:500] import os if "MEDIATUM_EMBED_IPYTHON" in os.environ: import IPython IPython.embed() del all_objects return render_template("memstats.j2.jade", maskcache=maskcache_info, sessions=sessions_info, memory=memory_info, naturalsize=humanize.filesize.naturalsize)
def apply_order_by_for_sortfields(query, sortfields_to_comp, before=False): for sortfield, (order, _) in iteritems(sortfields_to_comp): if order == "desc": desc = not bool(before) else: desc = bool(before) expr = node_value_expression(sortfield) if desc: expr = expr.desc() # attributes can be NULL (means: attribute doesn't exist), so we must be careful about null ordering if not (sortfield.startswith("node.") or sortfield == "nodename"): if before: expr = expr.nullsfirst() else: expr = expr.nullslast() query = query.order_by(expr) return query
def getContent(req, ids): node = q(Node).get(ids[0]) if not node.has_write_access() or "admin" in current_user.hidden_edit_functions: req.setStatus(httpstatus.HTTP_FORBIDDEN) return req.getTAL("web/edit/edit.html", {}, macro="access_error") if req.params.get("type", "") == "addattr" and req.params.get("new_name", "") != "" and req.params.get("new_value", "") != "": attrname = req.form.get("new_name") attrvalue = req.form.get("new_value") if attrname.startswith("system."): if current_user.is_admin: node.system_attrs[attrname[7:]] = attrvalue else: # non-admin user may not add / change system attributes, silently ignore the request. # XXX: an error msg would be better logg.warn("denied writing a system attribute because user is not an admin user, node=%s attrname=%s current_user=%s", node.id, attrname, current_user.id) return httpstatus.HTTP_FORBIDDEN node.set(attrname, attrvalue) db.session.commit() logg.info("new attribute %s for node %s added", req.params.get("new_name", ""), node.id) for key in req.params.keys(): # update localread value of current node if key.startswith("del_localread"): node.resetLocalRead() logg.info("localread attribute of node %s updated", node.id) break # removing attributes only allowed for admin user # remove attribute if key.startswith("attr_"): if not current_user.is_admin: return httpstatus.HTTP_FORBIDDEN del node.attrs[key[5:-2]] db.session.commit() logg.info("attribute %s of node %s removed", key[5:-2], node.id) break # remove system attribute if key.startswith("system_attr_"): if not current_user.is_admin: return httpstatus.HTTP_FORBIDDEN attrname = key[12:-2] del node.system_attrs[attrname] db.session.commit() logg.info("system attribute %s of node %s removed", attrname, node.id) break metadatatype = node.metadatatype fieldnames = [] if metadatatype: fields = metadatatype.getMetaFields() for field in fields: fieldnames += [field.name] else: fields = [] metafields = OrderedDict() technfields = OrderedDict() obsoletefields = OrderedDict() system_attrs = [] tattr = {} with suppress(AttributeError, warn=False): tattr = node.getTechnAttributes() tattr = formatTechAttrs(tattr) for key, value in sorted(iteritems(node.attrs), key=lambda t: t[0].lower()): if value or current_user.is_admin: # display all values for admins, even if they are "empty" (= a false value) if key in fieldnames: metafields[key] = formatdate(value, getFormat(fields, key)) elif key in tattr.keys(): technfields[key] = formatdate(value) else: obsoletefields[key] = value for key, value in sorted(iteritems(node.system_attrs), key=lambda t: t[0].lower()): system_attrs.append((key, value)) # remove all technical attributes if req.params.get("type", "") == "technical": for key in technfields: del node.attrs[key] technfields = {} logg.info("technical attributes of node %s removed", node.id) return req.getTAL("web/edit/modules/admin.html", {"id": req.params.get("id", "0"), "tab": req.params.get("tab", ""), "node": node, "obsoletefields": obsoletefields, "metafields": metafields, "system_attrs": system_attrs, "fields": fields, "technfields": technfields, "tattr": tattr, "fd": formatdate, "gf": getFormat, "user_is_admin": current_user.is_admin, "canedit": node.has_write_access(), "csrf": req.csrf_token.current_token}, macro="edit_admin_file")
def add_node_to_xmldoc(node, xmlroot, written=set(), children=True, exclude_filetypes=[], exclude_childtypes=[], attribute_name_filter=None): from schema.schema import Mask from schema.mapping import Mapping written.add(node.id) xmlnode = etree.SubElement(xmlroot, "node") xmlnode.set("name", node.name or u"") xmlnode.set("id", unicode(node.id)) xmlnode.set("type", (node.type + "/" + (node.schema or u"")).strip("/")) xmlnode.set("datatype", node.type) xmlnode.set("schema", (node.schema or u"")) # TODO: no access rights at the moment for name, value in sorted(iteritems(node.attrs)): if attribute_name_filter and not attribute_name_filter(name): continue xmlattr = etree.SubElement(xmlnode, "attribute") xmlattr.set("name", name) # protect XML from invalid characters # XXX: is this ok? xmlattr.text = etree.CDATA(xml_remove_illegal_chars(unicode(value))) files = [f for f in node.file_objects if f.filetype != u"metadata"] if exclude_filetypes: files = [f for f in files if f.filetype not in (exclude_filetypes)] for fileobj in files: add_file_to_xmlnode(fileobj, xmlnode) if children: child_query = node.children if exclude_childtypes: child_query = child_query.filter( ~Node.type.in_(exclude_childtypes)) for child in child_query.order_by("orderpos"): add_child_to_xmlnode(child, xmlnode) if child.id not in written: add_node_to_xmldoc(child, xmlroot, written, children, exclude_filetypes, exclude_childtypes, attribute_name_filter) if isinstance(node, Mask): exportmapping_id = node.get(u"exportmapping").strip() if exportmapping_id and exportmapping_id not in written: mapping = q(Mapping).get(int(exportmapping_id)) if mapping is not None: written.add(mapping.id) add_node_to_xmldoc(mapping, xmlroot, written, children, exclude_filetypes, exclude_childtypes, attribute_name_filter) return xmlnode
def convert_symbolic_rules_to_dnf(nid_to_symbolic_rule, simplify=False): return {nid: boolalg.to_dnf(rule, simplify=simplify) for nid, rule in iteritems(nid_to_symbolic_rule)}
def attributes_filtered_by_key(self, attribute_key_filter): return { k: v for k, v in iteritems(self.attributes) if attribute_key_filter(k) }
def getContent(req, ids): node = q(Node).get(ids[0]) if not node.has_write_access() or "admin" in current_user.hidden_edit_functions: req.setStatus(httpstatus.HTTP_FORBIDDEN) return req.getTAL("web/edit/edit.html", {}, macro="access_error") if req.params.get("type", "") == "addattr" and req.params.get("new_name", "") != "" and req.params.get("new_value", "") != "": attrname = req.form.get("new_name") attrvalue = req.form.get("new_value") if attrname.startswith("system."): if current_user.is_admin: node.system_attrs[attrname[7:]] = attrvalue else: # non-admin user may not add / change system attributes, silently ignore the request. # XXX: an error msg would be better logg.warn("denied writing a system attribute because user is not an admin user, node=%s attrname=%s current_user=%s", node.id, attrname, current_user.id) return httpstatus.HTTP_FORBIDDEN node.set(attrname, attrvalue) db.session.commit() logg.info("new attribute %s for node %s added", req.params.get("new_name", ""), node.id) for key in req.params.keys(): # update localread value of current node if key.startswith("del_localread"): node.resetLocalRead() logg.info("localread attribute of node %s updated", node.id) break # removing attributes only allowed for admin user # remove attribute if key.startswith("attr_"): if not current_user.is_admin: return httpstatus.HTTP_FORBIDDEN del node.attrs[key[5:-2]] db.session.commit() logg.info("attribute %s of node %s removed", key[5:-2], node.id) break # remove system attribute if key.startswith("system_attr_"): if not current_user.is_admin: return httpstatus.HTTP_FORBIDDEN attrname = key[12:-2] del node.system_attrs[attrname] db.session.commit() logg.info("system attribute %s of node %s removed", attrname, node.id) break metadatatype = node.metadatatype fieldnames = [] if metadatatype: fields = metadatatype.getMetaFields() for field in fields: fieldnames += [field.name] else: fields = [] metafields = OrderedDict() technfields = OrderedDict() obsoletefields = OrderedDict() system_attrs = [] tattr = {} try: tattr = node.getTechnAttributes() except AttributeError: pass tattr = formatTechAttrs(tattr) for key, value in sorted(iteritems(node.attrs), key=lambda t: t[0].lower()): if value or current_user.is_admin: # display all values for admins, even if they are "empty" (= a false value) if key in fieldnames: metafields[key] = formatdate(value, getFormat(fields, key)) elif key in tattr.keys(): technfields[key] = formatdate(value) else: obsoletefields[key] = value for key, value in sorted(iteritems(node.system_attrs), key=lambda t: t[0].lower()): system_attrs.append((key, value)) # remove all technical attributes if req.params.get("type", "") == "technical": for key in technfields: del node.attrs[key] technfields = {} logg.info("technical attributes of node %s removed", node.id) return req.getTAL("web/edit/modules/admin.html", {"id": req.params.get("id", "0"), "tab": req.params.get("tab", ""), "node": node, "obsoletefields": obsoletefields, "metafields": metafields, "system_attrs": system_attrs, "fields": fields, "technfields": technfields, "tattr": tattr, "fd": formatdate, "gf": getFormat, "user_is_admin": current_user.is_admin, "canedit": node.has_write_access()}, macro="edit_admin_file")
def statements_with_time(self): return ((k, v[0]) for k, v in iteritems(self._statements))
class Image(Content): #: create zoom tiles when width or height of image exceeds this value ZOOM_SIZE = 2000 ZOOM_TILESIZE = 256 # image formats that should exist for each mimetype of the `original` image IMAGE_FORMATS_FOR_MIMETYPE = defaultdict( lambda: [u"image/png"], { u"image/tiff": [u"image/tiff", u"image/png"], u"image/svg+xml": [u"image/svg+xml", u"image/png"], u"image/jpeg": [u"image/jpeg"], u"image/gif": [u"image/gif"], u"image/png": [u"image/png"], u"image/bmp": [u"image/bmp"] }) MIMETYPE_FOR_EXTENSION = { u"jpg": u"image/jpeg", u"jpeg": u"image/jpeg", u"png": u"image/png", u"tif": u"image/tiff", u"tiff": u"image/tiff", u"gif": u"image/gif", u"svg": u"image/svg+xml", u"bmp": u"image/bmp", } # beware of duplicates! EXTENSION_FOR_MIMETYPE = {v:k for k, v in iteritems(MIMETYPE_FOR_EXTENSION)} @classmethod def get_default_edit_menu_tabs(cls): return "menulayout(view);menumetadata(metadata;files;admin);menuclasses(classes);menusecurity(acls)" @classmethod def get_sys_filetypes(cls): return [u"original", u"thumb", u"image", u"presentation", u"zoom"] @classmethod def get_upload_filetype(cls): return u"original" @property def svg_image(self): return self.files.filter_by(filetype=u"image", mimetype=u"image/svg+xml").scalar() @property def zoom_available(self): zoom_file = self.files.filter_by(filetype=u"zoom").scalar() return zoom_file is not None @property def should_use_zoom(self): # svg should never use the flash zoom if self.svg_image is not None: return False return int(self.get("width") or 0) > Image.ZOOM_SIZE or int(self.get("height") or 0) > Image.ZOOM_SIZE def image_url_for_mimetype(self, mimetype): try: file_ext = Image.EXTENSION_FOR_MIMETYPE[mimetype] except KeyError: raise ValueError("unsupported image mimetype " + mimetype) url = u"/image/{}.{}".format(self.id, file_ext) return self._add_version_tag_to_url(url) @property def preferred_image_url(self): url = u"/image/" + unicode(self.id) return self._add_version_tag_to_url(url) @property def presentation_url(self): url = u"/thumb2/" + unicode(self.id) return self._add_version_tag_to_url(url) def get_image_formats(self): image_files = self.files.filter_by(filetype=u"image") image_formats = {} for img_file in image_files: if img_file.exists: image_formats[img_file.mimetype] = { "url": self.image_url_for_mimetype(img_file.mimetype), "display_size": humanize.filesize.naturalsize(img_file.size) } return image_formats # prepare hash table with values for TAL-template def _prepareData(self, req): obj = prepare_node_data(self, req) if obj["deleted"]: # no more processing needed if this object version has been deleted # rendering has been delegated to current version return obj obj["highres_url"] = None can_see_original = self.has_data_access() use_flash_zoom = config.getboolean("image.use_flash_zoom", True) and self.should_use_zoom image_url = '/fullsize?id=%d' % self.id if use_flash_zoom else '/image/%d' % self.id image_url = self._add_version_tag_to_url(image_url) archive = get_archive_for_node(self) if archive: if can_see_original: obj['highres_url'] = u"/file/{nid}/{nid}.tif".format(nid=self.id) archive_state = archive.get_file_state(self) if archive_state == Archive.NOT_PRESENT: obj['archive_fetch_url'] = u"/archive/{}".format(self.id) elif archive_state == Archive.PENDING: obj['archive_fetch_url'] = u"pending" elif archive_state == Archive.PRESENT: obj['archive_fetch_url'] = None files, sum_size = filebrowser(self, req) obj['canseeoriginal'] = can_see_original obj['preferred_image_url'] = self.preferred_image_url obj["image_formats"] = self.get_image_formats() obj['zoom'] = self.zoom_available obj['image_url'] = image_url obj['attachment'] = files obj['sum_size'] = sum_size obj['presentation_url'] = self.presentation_url obj['fullsize'] = str(self.id) if not self.isActiveVersion(): obj['tag'] = self.tag obj['fullsize'] += "&v=" + self.tag obj['fullsize'] = '"' + obj['fullsize'] + '"' full_style = req.args.get(u"style", u"full_standard") if full_style: obj['style'] = full_style return obj def _generate_other_format(self, mimetype_to_generate, files=None): original_file = filter_scalar(lambda f: f.filetype == u"original", files) extension = mimetype_to_generate.split("/")[1] newimg_name = os.path.splitext(original_file.abspath)[0] + "." + extension assert original_file.abspath != newimg_name if original_file.mimetype == u"image/svg+xml": convert_options = ["-alpha", "off", "-colorspace", "RGB", "-background", "white"] else: convert_options = [] old_file = filter_scalar(lambda f: f.filetype == u"image" and f.mimetype == mimetype_to_generate, files) if old_file is not None: self.files.remove(old_file) old_file.unlink() convert_image(original_file.abspath, newimg_name, convert_options) self.files.append(File(newimg_name, u"image", mimetype_to_generate)) def _check_missing_image_formats(self, files=None): if files is None: files = self.files.all() original_file = filter_scalar(lambda f: f.filetype == u"original", files) old_image_files = filter(lambda f: f.filetype == u"image", files) wanted_mimetypes = set(Image.IMAGE_FORMATS_FOR_MIMETYPE[original_file.mimetype]) return wanted_mimetypes - {f.mimetype for f in old_image_files} def _generate_image_formats(self, files=None, mimetypes_to_consider=None): """Creates other full size formats for this image node. TIFF: create new PNG to be used as `image` SVG: create PNG and add it as `png_image` :param mimetypes_to_consider: limit the formats that should be (re)-generated to this sequence of mimetypes """ if files is None: files = self.files.all() original_file = filter_scalar(lambda f: f.filetype == u"original", files) old_image_files = filter(lambda f: f.filetype == u"image", files) for old_img_file in old_image_files: # we don't want to remove the original file... if old_img_file.path != original_file.path: self.files.remove(old_img_file) old_img_file.unlink() mimetypes_to_generate = set(Image.IMAGE_FORMATS_FOR_MIMETYPE[original_file.mimetype]) if mimetypes_to_consider is not None: mimetypes_to_generate = mimetypes_to_generate.intersection(mimetypes_to_consider) for new_mimetype in mimetypes_to_generate: if new_mimetype == original_file.mimetype: # image is alias for the original image in this case fileobj = File(original_file.path, u"image", original_file.mimetype) self.files.append(fileobj) else: self._generate_other_format(new_mimetype, files) def _find_processing_file(self, files=None): """Finds the file that should be used for processing (generating thumbnails, extracting metadata etc) in a file sequence. """ if files is None: files = self.files.all() original_file = filter_scalar(lambda f: f.filetype == u"original", files) if original_file.mimetype == u"image/svg+xml": return filter_scalar(lambda f: f.filetype == u"image" and f.mimetype == u"image/png", files) return original_file def _generate_thumbnails(self, files=None): if files is None: files = self.files.all() image_file = self._find_processing_file(files) path = os.path.splitext(image_file.abspath)[0] # XXX: we really should use the correct file ending and find another way of naming thumbname = path + ".thumb" thumbname2 = path + ".presentation" old_thumb_files = filter(lambda f: f.filetype in (u"thumb", u"presentation"), files) # XXX: removing files before the new ones are created is bad, that should happen later (use File.unlink_after_deletion). # XXX: But we need better thumbnail naming first. for old in old_thumb_files: self.files.remove(old) old.unlink() make_thumbnail_image(image_file.abspath, thumbname) make_presentation_image(image_file.abspath, thumbname2) self.files.append(File(thumbname, u"thumb", u"image/jpeg")) self.files.append(File(thumbname2, u"presentation", u"image/jpeg")) def _generate_zoom_archive(self, files=None): if files is None: files = self.files.all() image_file = self._find_processing_file(files) zip_filename = get_zoom_zip_filename(self.id) zip_filepath = os.path.join(config.get("paths.zoomdir"), zip_filename) old_zoom_files = filter(lambda f: f.filetype == u"zoom", files) for old in old_zoom_files: self.files.remove(old) old.unlink() _create_zoom_archive(Image.ZOOM_TILESIZE, image_file.abspath, zip_filepath) file_obj = File(path=zip_filepath, filetype=u"zoom", mimetype=u"application/zip") self.files.append(file_obj) def _extract_metadata(self, files=None): image_file = self._find_processing_file(files) width, height = get_image_dimensions(image_file) # XXX: this is a bit redundant... self.set("origwidth", width) self.set("origheight", height) self.set("origsize", image_file.size) self.set("width", width) self.set("height", height) # Exif unwanted_attrs = Image.get_unwanted_exif_attributes() with open(image_file.abspath, 'rb') as f: tags = EXIF.process_file(f) for k in tags.keys(): # don't set unwanted exif attributes if any(tag in k for tag in unwanted_attrs): continue if tags[k]: self.set("exif_" + k.replace(" ", "_"), utf8_decode_escape(str(tags[k]))) # IPTC iptc_metadata = lib.iptc.IPTC.get_iptc_tags(image_file.abspath) if iptc_metadata is not None: for k, v in iteritems(iptc_metadata): self.set('iptc_' + k, v) def event_files_changed(self): """postprocess method for object type 'image'. called after object creation""" logg.debug("Postprocessing node %s", self.id) existing_files = self.files.all() if filter_scalar(lambda f: f.filetype == u"original", existing_files) is None: # we cannot do anything without an `original` file, stop here return missing_image_mimetypes = self._check_missing_image_formats(existing_files) if missing_image_mimetypes: self._generate_image_formats(existing_files, missing_image_mimetypes) # _generate_image_formats is allowed to change `image` and `original` images, so files = self.files.all() # generate both thumbnail sizes if one is missing because they should always display the same if (filter_scalar(lambda f: f.filetype == u"thumb", files) is None or filter_scalar(lambda f: f.filetype == u"presentation", files) is None): self._generate_thumbnails(files) # should we skip this sometimes? Do we want to overwrite everything? self._extract_metadata(files) if self.should_use_zoom: try: self._generate_zoom_archive(files) except: # XXX: this sometimes throws SystemError, see #806 # XXX: missing zoom tiles shouldn't abort the upload process logg.exception("zoom image generation failed!") # XXX: IPTC writeback will be fixed in #782 # self._writeback_iptc() db.session.commit() @classmethod def get_unwanted_exif_attributes(cls): ''' Returns a list of unwanted exif tags which are not to be extracted from uploaded images @return: list ''' return ['BitsPerSample', 'IPTC/NAA', 'WhitePoint', 'YCbCrCoefficients', 'ReferenceBlackWhite', 'PrimaryChromaticities', 'ImageDescription', 'StripOffsets', 'StripByteCounts', 'CFAPattern', 'CFARepeatPatternDim', 'YCbCrSubSampling', 'Tag', 'TIFFThumbnail', 'JPEGThumbnail', 'Thumbnail_BitsPerSample', 'GPS', 'CVAPattern', 'ApertureValue', 'ShutterSpeedValue', 'MakerNote', 'jpg_comment', 'UserComment', 'FlashPixVersion', 'ExifVersion', 'Caption', 'Byline', 'notice'] """ list with technical attributes for type image """ def getTechnAttributes(self): return {"Standard": {"creator": "Ersteller", "creationtime": "Erstelldatum", "updateuser": "******", "updatetime": "Update Datum", "updatesearchindex": "Update Suche", "height": "Höhe Thumbnail", "width": "Breite Thumbnail", "faulty": "Fehlerhaft", "workflow": "Workflownummer", "workflownode": "Workflow Knoten", "origwidth": "Originalbreite", "origheight": "Originalhöhe", "origsize": "Dateigröße"}, "Exif": {"exif_EXIF_ComponentsConfiguration": "EXIF ComponentsConfiguration", "exif_EXIF_LightSource": "EXIF LightSource", "exif_EXIF_FlashPixVersion": "EXIF FlashPixVersion", "exif_EXIF_ColorSpace": "EXIF ColorSpace", "exif_EXIF_MeteringMode": "EXIF MeteringMode", "exif_EXIF_ExifVersion": "EXIF ExifVersion", "exif_EXIF_Flash": "EXIF Flash", "exif_EXIF_DateTimeOriginal": "EXIF DateTimeOriginal", "exif_EXIF_InteroperabilityOffset": "EXIF InteroperabilityOffset", "exif_EXIF_FNumber": "EXIF FNumber", "exif_EXIF_FileSource": "EXIF FileSource", "exif_EXIF_ExifImageLength": "EXIF ExifImageLength", "exif_EXIF_SceneType": "EXIF SceneType", "exif_EXIF_CompressedBitsPerPixel": "EXIF CompressedBitsPerPixel", "exif_EXIF_ExposureBiasValue": "EXIF ExposureBiasValue", "exif_EXIF_ExposureProgram": "EXIF ExposureProgram", "exif_EXIF_ExifImageWidth": "EXIF ExifImageWidth", "exif_EXIF_DateTimeDigitized": "EXIF DateTimeDigitized", "exif_EXIF_FocalLength": "EXIF FocalLength", "exif_EXIF_ExposureTime": "EXIF ExposureTime", "exif_EXIF_ISOSpeedRatings": "EXIF ISOSpeedRatings", "exif_EXIF_MaxApertureValue": "EXIF MaxApertureValue", "exif_Image_Model": "Image Model", "exif_Image_Orientation": "Image Orientation", "exif_Image_DateTime": "Image DateTime", "exif_Image_YCbCrPositioning": "Image YCbCrPositioning", "exif_Image_ImageDescription": "Image ImageDescription", "exif_Image_ResolutionUnit": "Image ResolutionUnit", "exif_Image_XResolution": "Image XResolution", "exif_Image_Make": "Image Make", "exif_Image_YResolution": "Image YResolution", "exif_Image_ExifOffset": "Image ExifOffset", "exif_Thumbnail_ResolutionUnit": "Thumbnail ResolutionUnit", "exif_Thumbnail_DateTime": "Thumbnail DateTime", "exif_Thumbnail_JPEGInterchangeFormat": "Thumbnail JPEGInterchangeFormat", "exif_Thumbnail_JPEGInterchangeFormatLength": "Thumbnail JPEGInterchangeFormatLength", "exif_Thumbnail_YResolution": "Thumbnail YResolution", "exif_Thumbnail_Compression": "Thumbnail Compression", "exif_Thumbnail_Make": "Thumbnail Make", "exif_Thumbnail_XResolution": "Thumbnail XResolution", "exif_Thumbnail_Orientation": "Thumbnail Orientation", "exif_Thumbnail_Model": "Thumbnail Model", "exif_JPEGThumbnail": "JPEGThumbnail", "Thumbnail": "Thumbnail"}} """ fullsize popup-window for image node """ def popup_fullsize(self, req): # XXX: should be has_data_access instead, see #1135> # but cannot be changed at the moment if not self.has_read_access(): return 404 d = {} d["image_url"] = self.preferred_image_url d['tileurl'] = "/tile/{}/".format(self.id) d["no_flash_url"] = "/fullsize?id={}&no_flash=1".format(self.id) html = webconfig.theme.render_macro("image.j2.jade", "imageviewer", d) req.write(html) def popup_thumbbig(self, req): self.popup_fullsize(req) def processImage(self, type="", value="", dest=""): """XXX: this method is only called in shoppingbags. What does it even do?! """ img = None for file in self.files: if file.filetype == "image": img = file break if img: pic = PILImage.open(img.abspath) pic.load() if type == "percentage": w = pic.size[0] * int(value) / 100 h = pic.size[1] * int(value) / 100 if type == "pixels": if pic.size[0] > pic.size[1]: w = int(value) h = pic.size[1] * int(value) / pic.size[0] else: h = int(value) w = pic.size[0] * int(value) / pic.size[1] elif type == "standard": w, h = value.split("x") w = int(w) h = int(h) if pic.size[0] < pic.size[1]: factor_w = w * 1.0 / pic.size[0] factor_h = h * 1.0 / pic.size[1] if pic.size[0] * factor_w < w and pic.size[1] * factor_w < h: w = pic.size[0] * factor_w h = pic.size[1] * factor_w else: w = pic.size[0] * factor_h h = pic.size[1] * factor_h else: factor_w = w * 1.0 / pic.size[0] factor_h = h * 1.0 / pic.size[1] if pic.size[0] * factor_w < w and pic.size[1] * factor_w < h: w = pic.size[0] * factor_h h = pic.size[1] * factor_h else: w = pic.size[0] * factor_w h = pic.size[1] * factor_w else: # do nothing but copy image w = pic.size[0] h = pic.size[1] pic = pic.resize((int(w), int(h)), PILImage.ANTIALIAS) if not os.path.isdir(dest): os.mkdir(dest) pic.save(dest + self.id + ".jpg", "jpeg") return 1 return 0 def event_metadata_changed(self): pass # XXX: IPTC writeback will be fixed in #782 # self._writeback_iptc() def _writeback_iptc(self): """ Handles metadata content if changed. Creates a 'new' original [old == upload]. """ upload_file = None original_path = None original_file = None for f in self.files: if f.getType() == 'original': original_file = f if os.path.exists(f.abspath): original_path = f.abspath if os.path.basename(original_path).startswith('-'): return if f.type == 'upload': if os.path.exists(f.abspath): upload_file = f if not original_file: logg.info('No original upload for writing IPTC.') return if not upload_file: upload_path = '{}_upload{}'.format(os.path.splitext(original_path)[0], os.path.splitext(original_path)[-1]) import shutil shutil.copy(original_path, upload_path) self.files.append(File(upload_path, "upload", original_file.mimetype)) db.session.commit() tag_dict = {} for field in self.getMetaFields(): if field.get('type') == "meta" and field.getValueList()[0] != '' and 'on' in field.getValueList(): tag_name = field.getValueList()[0].split('iptc_')[-1] field_value = self.get('iptc_{}'.format(field.getName())) if field.getValueList()[0] != '' and 'on' in field.getValueList(): tag_dict[tag_name] = field_value lib.iptc.IPTC.write_iptc_tags(original_path, tag_dict)
def search_link(self, mode="simple"): params = {k: v for k, v in iteritems(self.url_params) if k not in ("query", "searchmode", "id")} if mode != "simple": params["searchmode"] = mode return node_url(self.container.id, **params) if not self.edit else edit_node_url(self.container.id, **params)
def statements_with_all_infos(self): return iteritems(self._statements)
def update(self, **kwargs): for name, value in iteritems(kwargs): setattr(self, name, value)
def attributes_filtered_by_value(self, attribute_value_filter): return { k: v for k, v in iteritems(self.attributes) if attribute_value_filter(v) }
def attributes_filtered_by_key(self, attribute_key_filter): return {k: v for k, v in iteritems(self.attributes) if attribute_key_filter(k)}
def attributes_filtered_by_value(self, attribute_value_filter): return {k: v for k, v in iteritems(self.attributes) if attribute_value_filter(v)}
def get_user_data(self, data): return {attribute_name: data.get(ldap_fieldname)[0].decode("utf8") if data.get(ldap_fieldname) else None for attribute_name, ldap_fieldname in iteritems(self.user_attributes)}
def get_maskcache_report(maskcache_accesscount): sorted_entries = [(k, v) for k, v in sorted(iteritems(maskcache_accesscount))] total_access_count = sum(v for k, v in sorted_entries) num_cache_keys = len(sorted_entries) return "keys: %s, total access count: %s, %s" % (num_cache_keys, total_access_count, sorted_entries)
def convert_symbolic_rules_to_dnf(nid_to_symbolic_rule, simplify=False): return { nid: boolalg.to_dnf(rule, simplify=simplify) for nid, rule in iteritems(nid_to_symbolic_rule) }