Пример #1
0
    def getFormattedValue(self,
                          metafield,
                          maskitem,
                          mask,
                          node,
                          language,
                          html=True,
                          template_from_caller=None):

        value = node.get_special(metafield.name)
        # consider int, long values like filesize
        if isinstance(value, (int, long)):
            value = str(value)
        value = value.replace(";", "; ")

        # ignore trailing newlines for textfields
        value = value.rstrip("\r\n")

        if value.find('\n') != -1:
            valuesList = value.split('\n')
            if any(lang in valuesList
                   for lang in config.languages):  # treat as multilingual
                index = 0
                try:
                    index = valuesList.index(language)
                    value = valuesList[index + 1]
                except ValueError as e:
                    msg = "Exception in getFormattedValue for textfield:\n"
                    msg += " valuesList=%r\n" % valuesList
                    msg += " node.name=%r, node.id=%r, node.type=%r\n" % (
                        node.name, node.id, node.type)
                    msg += " metafield.name=%r, metafield.id=%r, metafield.type=%r\n" % (
                        metafield.name, metafield.id, metafield.type)
                    msg += " language=%r, mask=%r" % (language, mask)
                    logg.exception(msg)

                    value = u""
            else:
                # treat as monolingual
                pass

        if html:
            value = esc(value)

        # replace variables
        # substitute TeX sub/super-scripts with <sub>/<sup> html tags
        value = modify_tex(value, 'html')

        value = replace_attribute_variables(value, node.id, node.get,
                                            r'&lt;(.+?)&gt;', "&lt;", "&gt;")
        value = value.replace("&lt;", "<").replace("&gt;", ">")

        if not maskitem:
            return (metafield.getLabel(), value)

        # use default value from mask if value is empty
        if value == u'':
            value = maskitem.getDefault()

        return (metafield.getLabel(), value)
Пример #2
0
    def getViewHTML(self, fields, nodes, flags, language="", template_from_caller=None, mask=None):
        ret = ""
        node = nodes[0]

        mask = fields[0].parents.first()
        separator = ""

        if mask.getMappingHeader() != "":
            ret += mask.getMappingHeader() + "\r\n"

        field_vals = []

        for field in fields:
            attribute_nid = field.get("attribute", None)
            if attribute_nid is None:
                continue

            try:
                attribute_nid = int(attribute_nid)
            except ValueError:
                logg.warn("ignoring field # %s with invalid attribute id: '%r'", field.id, attribute_nid)
                continue

            attrnode = q(Node).get(attribute_nid)
            if attrnode is None:
                continue

            if field.get("fieldtype") == "mapping":  # mapping to mapping definition
                exportmapping_id = mask.get("exportmapping").split(";")[0]
                mapping = q(Node).get(exportmapping_id)
                if mapping is None:
                    logg.warn("exportmapping %s for mask %s not found", exportmapping_id, mask.id)
                    return u""
                separator = mapping.get("separator")

                ns = mapping.getNamespace()
                if ns != "":
                    ns += ":"
                fld = q(Node).get(field.get("mappingfield"))
                format = fld.getExportFormat()
                field_value = ns + fld.getName()
                default = fld.getDefault().strip()
            else:  # attributes of node
                format = field.get("mappingfield")
                field_value = ""
                default = ""

            field_vals.append(self.replaceVars(format, node, attrnode, field_value, options=mask.getExportOptions(), mask=mask, default=default))

        if not mask.hasExportOption("l"):
            ret += separator.join(field_vals)
        else:
            ret += u"".join(field_vals)

        if mask.getMappingFooter() != "":
            ret += "\r\n" + mask.getMappingFooter()

        ret = modify_tex(ret, 'strip')

        return self.replaceVars(ret, node, mask)
Пример #3
0
    def getViewHTML(self,
                    fields,
                    nodes,
                    flags,
                    language="",
                    template_from_caller=None,
                    mask=None):
        ret = ""
        node = nodes[0]

        mask = fields[0].getParents()[0]
        separator = ""

        if mask.getMappingHeader() != "":
            ret += mask.getMappingHeader() + "\r\n"

        for field in fields:
            try:
                attrnode = tree.getNode(field.get("attribute"))
            except:
                continue

            if field.get(
                    "fieldtype") == "mapping":  # mapping to mapping definition
                mapping = tree.getNode(mask.get("exportmapping").split(";")[0])
                separator = mapping.get("separator")

                ns = mapping.getNamespace()
                if ns != "":
                    ns += ":"
                fld = tree.getNode(field.get("mappingfield"))
                format = fld.getExportFormat()
                field_value = ns + fld.getName()
                default = fld.getDefault().strip()
            else:  # attributes of node
                format = field.get("mappingfield")
                field_value = ""
                default = ""

            ret += self.replaceVars(format,
                                    node,
                                    attrnode,
                                    field_value,
                                    options=mask.getExportOptions(),
                                    mask=mask,
                                    default=default)

            if not mask.hasExportOption(
                    "l") and list(fields).index(field) < len(fields) - 1:
                ret += separator

        if mask.getMappingFooter() != "":
            ret += "\r\n" + mask.getMappingFooter()

        ret = modify_tex(ret, 'strip')

        return self.replaceVars(ret, node, mask)
Пример #4
0
    def nodeToExtSearch(self, node,
                        schema):  # build extended search index from node

        if len(node.getSearchFields()
               ) == 0:  # stop if schema has no searchfields
            return True

        self.nodeToSchemaDef(node, schema)  # save definition

        keyvalue = []
        i = 1
        for field in node.getSearchFields():
            key = "field%d" % i
            i += 1
            value = ""
            if field.getFieldtype() == "union":
                for item in field.get("valuelist").split(";"):
                    value += node.get(item) + '|'
            else:
                value = node.get(field.getName())
            keyvalue += [(key, modify_tex(u(protect(value)), 'strip'))]

        sql0 = 'SELECT id FROM searchmeta where id=\'{}\''.format(node.id)
        sql1 = 'UPDATE searchmeta SET '
        sql2 = 'INSERT INTO searchmeta (id, type, schema, updatetime'
        for key, value in keyvalue:
            sql1 += key + "='" + normalize_utf8(value) + "', "
            sql2 += ", "
            sql2 += key
        sql1 += "type='" + node.getContentType(
        ) + "', schema='" + schema + "', updatetime='" + node.get(
            "updatetime") + "'"
        sql2 += ") VALUES("
        sql2 += '\'{}\', "{}", "{}", "{}"'.format(node.id,
                                                  node.getContentType(),
                                                  schema,
                                                  node.get("updatetime"))

        for key, value in keyvalue:
            sql2 += ", '" + normalize_utf8(value) + "'"
        sql1 += " WHERE id='{}'".format(node.id)
        sql2 += ")"

        sql = ""
        try:
            sql = sql0
            if self.execute(sql0, schema, 'ext'):  # select
                sql = sql1
                self.execute(sql1, schema, 'ext')  # do update
            else:
                sql = sql2
                self.execute(sql2, schema, 'ext')  # do insert
            return True
        except:
            logException('error in sqlite insert/update: ' + sql)
            return False
Пример #5
0
    def nodeToExtSearch(self, node, schema):  # build extended search index from node

        if len(node.getSearchFields()) == 0:  # stop if schema has no searchfields
            return True

        self.nodeToSchemaDef(node, schema)  # save definition

        keyvalue = []
        i = 1
        for field in node.getSearchFields():
            key = "field%d" % i
            i += 1
            value = ""
            if field.getFieldtype() == "union":
                for item in field.get("valuelist").split(";"):
                    value += node.get(item) + '|'
            else:
                value = node.get(field.getName())
            keyvalue += [(key, modify_tex(u(protect(value)), 'strip'))]

        sql0 = 'SELECT id FROM searchmeta where id=\'{}\''.format(node.id)
        sql1 = 'UPDATE searchmeta SET '
        sql2 = 'INSERT INTO searchmeta (id, type, schema, updatetime'
        for key, value in keyvalue:
            sql1 += key + "='" + normalize_utf8(value) + "', "
            sql2 += ", "
            sql2 += key
        sql1 += "type='" + node.getContentType() + "', schema='" + schema + "', updatetime='" + node.get("updatetime") + "'"
        sql2 += ") VALUES("
        sql2 += '\'{}\', "{}", "{}", "{}"'.format(node.id,
                                          node.getContentType(),
                                          schema,
                                          node.get("updatetime"))

        for key, value in keyvalue:
            sql2 += ", '" + normalize_utf8(value) + "'"
        sql1 += " WHERE id='{}'".format(node.id)
        sql2 += ")"

        sql = ""
        try:
            sql = sql0
            if self.execute(sql0, schema, 'ext'):  # select
                sql = sql1
                self.execute(sql1, schema, 'ext')  # do update
            else:
                sql = sql2
                self.execute(sql2, schema, 'ext')  # do insert
            return True
        except:
            logException('error in sqlite insert/update: ' + sql)
            return False
Пример #6
0
    def getFormattedValue(self, metafield, maskitem, mask, node, language, html=True, template_from_caller=None):

        value = node.get_special(metafield.name)
        # consider int, long values like filesize
        if isinstance(value, (int, long)):
            value = str(value)
        value = value.replace(";", "; ")

        # ignore trailing newlines for textfields
        value = value.rstrip("\r\n")

        if value.find('\n') != -1:
            valuesList = value.split('\n')
            if any(lang in valuesList for lang in config.languages):  # treat as multilingual
                index = 0
                try:
                    index = valuesList.index(language)
                    value = valuesList[index + 1]
                except ValueError as e:
                    msg = "Exception in getFormattedValue for textfield:\n"
                    msg += " valuesList=%r\n" % valuesList
                    msg += " node.name=%r, node.id=%r, node.type=%r\n" % (node.name, node.id, node.type)
                    msg += " metafield.name=%r, metafield.id=%r, metafield.type=%r\n" % (metafield.name, metafield.id, metafield.type)
                    msg += " language=%r, mask=%r" % (language, mask)
                    logg.exception(msg)

                    value = u""
            else:
                # treat as monolingual
                pass

        if html:
            value = esc(value)

        # replace variables
        # substitute TeX sub/super-scripts with <sub>/<sup> html tags
        value = modify_tex(value, 'html')

        value = replace_attribute_variables(value, node.id, node.get, r'&lt;(.+?)&gt;', "&lt;", "&gt;")
        value = value.replace("&lt;", "<").replace("&gt;", ">")

        if not maskitem:
            return (metafield.getLabel(), value)

        # use default value from mask if value is empty
        if value == u'':
            value = maskitem.getDefault()

        return (metafield.getLabel(), value)
Пример #7
0
    def nodeToSimpleSearch(self,
                           node,
                           schema,
                           type=""):  # build simple search index from node

        sql_upd = "UPDATE fullsearchmeta SET type='{}', schema='{}', value='{}| ".format(
            node.getContentType(), node.getSchema(), node.name)
        sql_ins = "INSERT INTO fullsearchmeta (id, type, schema, value) VALUES('{}', '{}', '{}', '{}| ".format(
            node.id, node.getContentType(), node.getSchema(), node.name)

        # attributes
        val = ''
        for key, value in node.items():
            if key not in SYSTEMATTRS:  # ignore system attributes
                val += protect(u(value)) + '| '
        for v in val.split(" "):
            v = u(v)
            if normalize_utf8(v) != v.lower():
                val += ' ' + normalize_utf8(v)

        val = val.replace(chr(0), "") + ' '

        # remove tex markup
        val = modify_tex(val, 'strip')

        # files
        for file in node.getFiles():
            val += protect(
                u(file.getName() + '| ' + file.getType() + '| ' +
                  file.getMimeType()) + '| ')

        sql_upd += val + '\' WHERE id=\'{}\''.format(node.id)
        sql_ins += val + '\')'

        sql = ""
        try:
            sql = 'SELECT id from fullsearchmeta WHERE id=\'{}\''.format(
                node.id)
            if self.execute(sql, schema, 'full'):  # check existance
                sql = sql_upd  # do update
            else:
                sql = sql_ins  # do insert
            self.execute(sql, schema, 'full')
            return True
        except:
            logException('error in sqlite insert/update: ' + sql)
            return False
Пример #8
0
    def getViewHTML(self, fields, nodes, flags, language="", template_from_caller=None, mask=None):
        ret = ""
        node = nodes[0]

        mask = fields[0].getParents()[0]
        separator = ""

        if mask.getMappingHeader() != "":
            ret += mask.getMappingHeader() + "\r\n"

        for field in fields:
            try:
                attrnode = tree.getNode(field.get("attribute"))
            except:
                continue

            if field.get("fieldtype") == "mapping":  # mapping to mapping definition
                mapping = tree.getNode(mask.get("exportmapping").split(";")[0])
                separator = mapping.get("separator")

                ns = mapping.getNamespace()
                if ns != "":
                    ns += ":"
                fld = tree.getNode(field.get("mappingfield"))
                format = fld.getExportFormat()
                field_value = ns + fld.getName()
                default = fld.getDefault().strip()
            else:  # attributes of node
                format = field.get("mappingfield")
                field_value = ""
                default = ""

            ret += self.replaceVars(
                format, node, attrnode, field_value, options=mask.getExportOptions(), mask=mask, default=default
            )

            if not mask.hasExportOption("l") and list(fields).index(field) < len(fields) - 1:
                ret += separator

        if mask.getMappingFooter() != "":
            ret += "\r\n" + mask.getMappingFooter()

        ret = modify_tex(ret, "strip")

        return self.replaceVars(ret, node, mask)
Пример #9
0
    def nodeToSimpleSearch(self, node, schema, type=""):  # build simple search index from node

        sql_upd ="UPDATE fullsearchmeta SET type='{}', schema='{}', value='{}| ".format(node.getContentType(),
                                                                                        node.getSchema(),
                                                                                        node.name)
        sql_ins = "INSERT INTO fullsearchmeta (id, type, schema, value) VALUES('{}', '{}', '{}', '{}| ".format(node.id,
                                                                                                               node.getContentType(),
                                                                                                               node.getSchema(),
                                                                                                               node.name)

        # attributes
        val = ''
        for key, value in node.items():
            if key not in SYSTEMATTRS:  # ignore system attributes
                val += protect(u(value)) + '| '
        for v in val.split(" "):
            v = u(v)
            if normalize_utf8(v) != v.lower():
                val += ' ' + normalize_utf8(v)

        val = val.replace(chr(0), "") + ' '

        # remove tex markup
        val = modify_tex(val, 'strip')

        # files
        for file in node.getFiles():
            val += protect(u(file.getName() + '| ' + file.getType() + '| ' + file.getMimeType()) + '| ')

        sql_upd += val + '\' WHERE id=\'{}\''.format(node.id)
        sql_ins += val + '\')'

        sql = ""
        try:
            sql = 'SELECT id from fullsearchmeta WHERE id=\'{}\''.format(node.id)
            if self.execute(sql, schema, 'full'):  # check existance
                sql = sql_upd  # do update
            else:
                sql = sql_ins  # do insert
            self.execute(sql, schema, 'full')
            return True
        except:
            logException('error in sqlite insert/update: ' + sql)
            return False
Пример #10
0
def get_extended_field_ratio(schema, node, db_content):
    """
    Compares the values in the ext search db and the values in the node instance and returns
    a ratio of likeness between the two values.
    @param schema: String, name of the schema
    @param node: Node, an core.tree node instance
    @return: Float
    """
    ratios = []

    field_names = get_zero_index_schema_fields(schema)

    for field in field_names:
        node_value = normalize_utf8(modify_tex(u(protect(node.get(field.name))), 'strip'))
        db_value = str(db_content[field.position])
        equality_ratio = difflib.SequenceMatcher(None, db_value, node_value).ratio()
        ratios.append(equality_ratio)

    return sum(ratios) / len(ratios)
Пример #11
0
def get_extended_field_ratio(schema, node, db_content):
    """
    Compares the values in the ext search db and the values in the node instance and returns
    a ratio of likeness between the two values.
    @param schema: String, name of the schema
    @param node: Node, an core.tree node instance
    @return: Float
    """
    ratios = []

    field_names = get_zero_index_schema_fields(schema)

    for field in field_names:
        node_value = normalize_utf8(
            modify_tex(u(protect(node.get(field.name))), 'strip'))
        db_value = str(db_content[field.position])
        equality_ratio = difflib.SequenceMatcher(None, db_value,
                                                 node_value).ratio()
        ratios.append(equality_ratio)

    return sum(ratios) / len(ratios)
Пример #12
0
    def getFormatedValue(self,
                         field,
                         node,
                         language=None,
                         html=1,
                         template_from_caller=None,
                         mask=None):

        value = node.get(field.getName()).replace(";", "; ")

        # ignore trailing newlines for textfields
        value = value.rstrip("\r\n")

        if value.find('\n') != -1:
            valuesList = value.split('\n')
            if any(lang in valuesList
                   for lang in system_languages):  # treat as multilingual
                index = 0
                try:
                    index = valuesList.index(language)
                    value = valuesList[index + 1]
                except ValueError as e:
                    logException(e)

                    log = logging.getLogger("errors")
                    msg = "Exception in getFormatedValue for textfield:\n"
                    msg += " valuesList=%r\n" % valuesList
                    msg += " node.name=%r, node.id=%r, node.type=%r\n" % (
                        node.name, node.id, node.type)
                    msg += " field.name=%r, field.id=%r, field.type=%r\n" % (
                        field.name, field.id, field.type)
                    msg += " language=%r, mask=%r" % (language, mask)
                    log.error(msg)

                    value = ""
            else:
                # treat as monolingual
                pass
        unescaped_value = value

        if html:
            value = esc(value)

        # replace variables
        # substitute TeX sub/super-scripts with <sub>/<sup> html tags
        value = modify_tex(value, 'html')

        for var in re.findall(r'&lt;(.+?)&gt;', value):
            if var == "att:id":
                value = value.replace("&lt;" + var + "&gt;", node.id)
            elif var.startswith("att:"):
                val = node.get(var[4:])
                if val == "":
                    val = "____"

                value = value.replace("&lt;" + var + "&gt;", val)
        value = value.replace("&lt;", "<").replace("&gt;", ">")

        maskitem = getMaskitemForField(field, language=language, mask=mask)
        if not maskitem:
            return (field.getLabel(), value)

        # use default value from mask if value is empty
        if value == '':
            value = maskitem.getDefault()

        if template_from_caller and template_from_caller[
                0] and maskitem and str(
                    maskitem.id) == template_from_caller[3]:
            value = template_from_caller[0]

        context = {'node': node, 'host': "http://" + config.get("host.name")}

        if (template_from_caller and
                template_from_caller[0]) and (not node.get(field.getName())):
            value = runTALSnippet(value, context)
        else:
            try:
                value = runTALSnippet(value, context)
            except:
                value = runTALSnippet(unescaped_value, context)

        return (field.getLabel(), value)
Пример #13
0
    def mkIndex(name, field, type, data=None):
        print "Making Index", name
        file = tmpdir + name + ".searchfile.txt"
        fi = open(file, "wb")

        for node in nodelist:
            if field == "alltext":
                s = ""
                if node.type != "directory":
                    for key, val in node.items():
                        if val:
                            s += val + "\n"
                    s += node.getName()

                # for nfile in node.getFiles():
                #    if nfile.type == "fulltext":
                #        try:
                #            fi2 = open(nfile.retrieveFile(), "rb")
                #            s += " "
                #            s += fi2.read()
                #            s += " "
                #            fi2.close()
                #        except IOError:
                #            log.error("Couldn't access file "+nfile.getName())
            elif field == "fulltext":
                s = ""
                if node.type != "directory":
                    for nfile in node.getFiles():
                        if nfile.type == "fulltext":
                            try:
                                fi2 = open(nfile.retrieveFile(), "rb")
                                s += " "
                                s += fi2.read()
                                s += " "
                                fi2.close()
                            except IOError:
                                log.error("Couldn't access file " +
                                          nfile.getName())

                #c1 = "abcdefghijklmnop"[(int(node.id)>>28)&15]
                #c2 = "abcdefghijklmnop"[(int(node.id)>>24)&15]
                #c3 = "abcdefghijklmnop"[(int(node.id)>>20)&15]
                #c4 = "abcdefghijklmnop"[(int(node.id)>>16)&15]
                #c5 = "abcdefghijklmnop"[(int(node.id)>>12)&15]
                #c6 = "abcdefghijklmnop"[(int(node.id)>>8)&15]
                #c7 = "abcdefghijklmnop"[(int(node.id)>>4)&15]
                #c8 = "abcdefghijklmnop"[(int(node.id)>>0)&15]
                #s += " mx" + c8+c7+c6+c5+c4+c3+c2+c1 + " "
            elif field == "everything":
                s = ""
                if node.type != "directory":
                    for key, val in node.items():
                        if val:
                            s += UTF8ToLatin(val) + "\n"
                    s += UTF8ToLatin(node.getName())
                    for nfile in node.getFiles():
                        if nfile.type == "fulltext":
                            try:
                                fi2 = open(nfile.retrieveFile(), "rb")
                                s += " "
                                s += fi2.read()
                                s += " "
                                fi2.close()
                            except IOError:
                                log.error("Couldn't access file " +
                                          nfile.getName())

            elif field == "objtype":
                s = node.type
                if "/" in s:
                    s = s[:s.index("/")]
            elif field == "schema":
                s = node.type
                if "/" in s:
                    s = s[s.index("/") + 1:]
                else:
                    s = ""
            elif type == "union":
                s = ""
                for item in data:
                    field = node.get(item)
                    if field:
                        s += field + " "
            else:
                s = node.get(field)
                if type == "date":
                    if len(s):
                        try:
                            s = str(date.parse_date(s).daynum())
                        except:
                            print "Couldn't parse date", s
                            s = "0"

            s = UTF8ToLatin(s)
            s = modify_tex(s, 'strip')
            fi.write(s.replace("\2", " ") + "\n")
            fi.write("\2")

        if type != "num" and type != "date":
            fi.write(
                "ecjfadf;jwkljer;jfklajd;gyugi;wyuogsdfjg;wuriosygh;nmwert;bwweriwoue;jfkajsdf;nmweurwu;hkethre;ghbyxuidfg;ewrioafi;ewirjglsag;vhxyseoru;vnmwerwe;fajsdfwetrh"
            )
        fi.close()

        basename = name

        itype = "makeindex"
        if type == "date" or type == "num":
            itype = "makeNumIndex"
        elif type == "list" or type == "mlist" or type == "ilist":
            itype = "makeClassIndex"

        command = "%s %s %s %s %s %s" % (
            sys.executable,
            os.path.join(config.basedir, "core/search/runindexer.py"), itype,
            file, tmpdir, basename)
        exit_status = os.system(command)
        if exit_status:
            print "Exit status " + str(
                exit_status) + " of subprocess " + command
            sys.exit(1)
Пример #14
0
    def getFormatedValue(self, field, node, language=None, html=1, template_from_caller=None, mask=None):

        value = node.get(field.getName()).replace(";", "; ")

        # ignore trailing newlines for textfields
        value = value.rstrip("\r\n")

        if value.find('\n') != -1:
            valuesList = value.split('\n')
            if any(lang in valuesList for lang in system_languages):  # treat as multilingual
                index = 0
                try:
                    index = valuesList.index(language)
                    value = valuesList[index + 1]
                except ValueError as e:
                    logException(e)

                    log = logging.getLogger("errors")
                    msg = "Exception in getFormatedValue for textfield:\n"
                    msg += " valuesList=%r\n" % valuesList
                    msg += " node.name=%r, node.id=%r, node.type=%r\n" % (node.name, node.id, node.type)
                    msg += " field.name=%r, field.id=%r, field.type=%r\n" % (field.name, field.id, field.type)
                    msg += " language=%r, mask=%r" % (language, mask)
                    log.error(msg)

                    value = ""
            else:
                # treat as monolingual
                pass
        unescaped_value = value

        if html:
            value = esc(value)

        # replace variables
        # substitute TeX sub/super-scripts with <sub>/<sup> html tags
        value = modify_tex(value, 'html')

        for var in re.findall(r'&lt;(.+?)&gt;', value):
            if var == "att:id":
                value = value.replace("&lt;" + var + "&gt;", node.id)
            elif var.startswith("att:"):
                val = node.get(var[4:])
                if val == "":
                    val = "____"

                value = value.replace("&lt;" + var + "&gt;", val)
        value = value.replace("&lt;", "<").replace("&gt;", ">")

        maskitem = getMaskitemForField(field, language=language, mask=mask)
        if not maskitem:
            return (field.getLabel(), value)

        # use default value from mask if value is empty
        if value == '':
            value = maskitem.getDefault()

        if template_from_caller and template_from_caller[0] and maskitem and str(maskitem.id) == template_from_caller[3]:
            value = template_from_caller[0]

        context = {'node': node, 'host': "http://" + config.get("host.name")}

        if (template_from_caller and template_from_caller[0]) and (not node.get(field.getName())):
            value = runTALSnippet(value, context)
        else:
            try:
                value = runTALSnippet(value, context)
            except:
                value = runTALSnippet(unescaped_value, context)

        return (field.getLabel(), value)
Пример #15
0
def write_formatted_response(req,
                             path,
                             params,
                             data,
                             id,
                             debug=True,
                             allchildren=False,
                             singlenode=False,
                             parents=False):

    atime = starttime = time.time()
    r_timetable = []

    _p = params.copy()
    if "jsoncallback" in _p:
        del _p['jsoncallback']
    if "_" in _p:
        del _p['_']

    send_children = "send_children" in params

    cache_key = '|'.join(
        map(str, [path, _p, allchildren, singlenode, parents, send_children]))
    cache_key = cache_key.replace(' ', '_')

    acceptcached = float(params.get('acceptcached', DEFAULT_CACHE_VALID))

    result_from_cache = None
    if acceptcached > 0.0:
        resultcode, cachecontent = resultcache.retrieve(
            cache_key, acceptcached)
        if resultcode == 'hit':
            cache_name = 'resultcache'
            timestamp_from_cache = time_cached = resultcache.getTimestamp(
                cache_key)
            time_delta = starttime - time_cached
            result_from_cache, mimetype_from_cache = cachecontent[-1]
            # replace jQuery, jsonp callback value
            if result_from_cache.startswith(
                    'jQuery') or result_from_cache.startswith('jsonp'):
                result_from_cache = params['jsoncallback'] + result_from_cache[
                    result_from_cache.find("({"):]

            r_timetable.append([
                "retrieved filtered result from 'resultcache': (%d bytes), time_delta: %.3f lower acceptcached %.3f sec."
                % (len(result_from_cache), time_delta, acceptcached),
                time.time() - atime
            ])
            atime = time.time()
        elif resultcode == 'refused':
            time_cached = resultcache.getTimestamp(cache_key)
            time_delta = starttime - time_cached
            r_timetable.append([
                "cached result exists in 'resultcache', but not used: time_delta: %.3f sec. higher acceptcached %.3f sec."
                % (time_delta, acceptcached),
                time.time() - atime
            ])
            atime = time.time()

    if not result_from_cache:

        res_format = (params.get('format', 'xml')).lower()

        # XXX: hack because we want all files for the XML format only
        if res_format == "xml":
            fetch_files = True
        else:
            fetch_files = False

        d = get_node_data_struct(req,
                                 path,
                                 params,
                                 data,
                                 id,
                                 debug=debug,
                                 allchildren=allchildren,
                                 singlenode=singlenode,
                                 send_children=send_children,
                                 parents=parents,
                                 fetch_files=fetch_files,
                                 csv=res_format == u'csv')

        if r_timetable:
            d['timetable'] = r_timetable + d.setdefault('timetable', [])
            r_timetable = []

        formatIsSupported = False

        for supported_format in supported_formats:
            if res_format in supported_format[0]:
                atime = time.time()
                formatIsSupported = True
                s = supported_format[1](req,
                                        path,
                                        params,
                                        data,
                                        d,
                                        debug=debug,
                                        singlenode=singlenode,
                                        send_children=send_children)
                if res_format == 'json' and 'jsoncallback' in params:
                    s = params['jsoncallback'] + '(' + s + ')'
                    # the return value of this kind of call must be interpreted as javascript,
                    # so we must set the mimetype or browsers will complain
                    mimetype = "application/javascript"
                else:
                    # XXX: clients can override the content_type by setting the mimetype param
                    # XXX: this is ugly, but we keep it for compatibility
                    mimetype = params.get('mimetype', supported_format[2])

                # append correct charset if client didn't force another value
                # it doesn't make sense to set it in the client to a different charset than utf8, but it was possible in the past...
                if "charset=" in mimetype:
                    content_type = mimetype
                else:
                    content_type = mimetype + "; charset=utf-8"

                d['timetable'].append(
                    ["formatted for '%s'" % res_format,
                     time.time() - atime])
                atime = time.time()

                disposition = params.get('disposition', '')
                if disposition:
                    # ex.: (direct to download) value: "attachment; filename=myfilename.txt"
                    # ex.: (open in browser) value: "filename=myfilename.txt"
                    req.reply_headers['Content-Disposition'] = disposition
                    d['timetable'].append([
                        "wrote disposition %r to reply header" % (disposition),
                        time.time() - atime
                    ])
                    atime = time.time()

                break

        if not formatIsSupported:
            d['status'] = 'fail'
            d['html_response_code'] = '404'  # not found
            d['errormessage'] = 'unsupported format'
            d['build_response_end'] = time.time()

            s = struct2xml(req,
                           path,
                           params,
                           data,
                           d,
                           singlenode=True,
                           send_children=False,
                           fetch_files=True)
            content_type = "text/xml; charset=utf-8"

        if acceptcached > 0:  # only write to cache for these requests
            resultcache.update(cache_key, [s, mimetype])
            d['timetable'].append([
                "wrote result to 'resultcache' (%d bytes), now in cache: %d entries"
                % (len(s), resultcache.getKeysCount()),
                time.time() - atime
            ])
            atime = time.time()
            #d['timetable'].append(["wrote result to 'resultcache' (%d bytes), now in cache: %d entries: \r\n%s" % (len(s), resultcache.getKeysCount(), resultcache.report()), time.time()-atime]); atime = time.time()
            # d['timetable'].append(["wrote result to 'resultcache' (%d bytes), now in
            # cache: %d entries: %s" % (len(s), resultcache.getKeysCount(),
            # "#".join(resultcache.getKeys())), time.time()-atime]); atime =
            # time.time()

        s = modify_tex(s.decode("utf8"), 'strip').encode("utf8")

    else:
        d = {}
        d['status'] = 'ok'
        d['html_response_code'] = '200'  # ok
        d['build_response_end'] = time.time()
        if r_timetable:
            d['timetable'] = r_timetable
        s = result_from_cache
        d['timetable'].append([
            "serving %.3f sec. old response (%d bytes) from '%s', cache_key: %s"
            % (time.time() - timestamp_from_cache, len(s), cache_name,
               cache_key),
            time.time() - atime
        ])
        atime = time.time()
        mimetype = mimetype_from_cache
        content_type = mimetype + "; charset=utf-8"
        req.reply_headers['Content-Type'] = content_type

    def compressForDeflate(s):
        import gzip
        return gzip.zlib.compress(s, 9)

    def compressForGzip(s):
        import cStringIO
        import gzip
        buffer = cStringIO.StringIO()
        gzfile = gzip.GzipFile(mode='wb', fileobj=buffer, compresslevel=9)
        gzfile.write(s)
        gzfile.close()
        return buffer.getvalue()

    if 'deflate' in params:
        size_uncompressed = len(s)
        compressed_s = compressForDeflate(s)
        s = compressed_s
        size_compressed = len(s)
        try:
            percentage = 100.0 * size_compressed / size_uncompressed
        except:
            percentage = 100.0
        req.reply_headers['Content-Encoding'] = "deflate"
        d['timetable'].append([
            "'deflate' in request: executed compressForDeflate(s), %d bytes -> %d bytes (compressed to: %.1f %%)"
            % (size_uncompressed, size_compressed, percentage),
            time.time() - atime
        ])
        atime = time.time()

    elif 'gzip' in params:
        size_uncompressed = len(s)
        compressed_s = compressForGzip(s)
        s = compressed_s
        size_compressed = len(s)
        try:
            percentage = 100.0 * size_compressed / size_uncompressed
        except:
            percentage = 100.0
        req.reply_headers['Content-Encoding'] = "gzip"
        d['timetable'].append([
            "'gzip' in request: executed compressForGzip(s), %d bytes -> %d bytes (compressed to: %.1f %%)"
            % (size_uncompressed, size_compressed, percentage),
            time.time() - atime
        ])
        atime = time.time()

    # (format) Expires: Mon, 28 Nov 2011 12:41:22 GMT
    # see core.athana.build_http_date
    # req.reply_headers['Expires'] = time.strftime ('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(time.time()+60.0)) # 1 minute

    # remark: on 2011-12-01 switched response from req.write to sendAsBuffer for performance reasons
    # (before: ) req.write(s)
    _sendAsBuffer(req,
                  s,
                  content_type,
                  force=1,
                  allow_cross_origin=allow_cross_origin)
    d['timetable'].append([
        "executed sendAsBuffer, %d bytes, content type='%s'" %
        (len(s), content_type),
        time.time() - atime
    ])
    atime = time.time()
    return d['html_response_code'], len(s), d
Пример #16
0
    def getViewHTML(self,
                    fields,
                    nodes,
                    flags,
                    language="",
                    template_from_caller=None,
                    mask=None):
        ret = ""
        node = nodes[0]

        mask = fields[0].parents.first()
        separator = ""

        if mask.getMappingHeader() != "":
            ret += mask.getMappingHeader() + "\r\n"

        field_vals = []

        for field in fields:
            attribute_nid = field.get("attribute", None)
            if attribute_nid is None:
                continue

            try:
                attribute_nid = int(attribute_nid)
            except ValueError:
                logg.warn(
                    "ignoring field # %s with invalid attribute id: '%r'",
                    field.id, attribute_nid)
                continue

            attrnode = q(Node).get(attribute_nid)
            if attrnode is None:
                continue

            if field.get(
                    "fieldtype") == "mapping":  # mapping to mapping definition
                exportmapping_id = mask.get("exportmapping").split(";")[0]
                mapping = q(Node).get(exportmapping_id)
                if mapping is None:
                    logg.warn("exportmapping %s for mask %s not found",
                              exportmapping_id, mask.id)
                    return u""
                separator = mapping.get("separator")

                ns = mapping.getNamespace()
                if ns != "":
                    ns += ":"
                fld = q(Node).get(field.get("mappingfield"))
                format = fld.getExportFormat()
                field_value = ns + fld.getName()
                default = fld.getDefault().strip()
            else:  # attributes of node
                format = field.get("mappingfield")
                field_value = ""
                default = ""

            field_vals.append(
                self.replaceVars(format,
                                 node,
                                 attrnode,
                                 field_value,
                                 options=mask.getExportOptions(),
                                 mask=mask,
                                 default=default))

        if not mask.hasExportOption("l"):
            ret += separator.join(field_vals)
        else:
            ret += u"".join(field_vals)

        if mask.getMappingFooter() != "":
            ret += "\r\n" + mask.getMappingFooter()

        ret = modify_tex(ret, 'strip')

        return self.replaceVars(ret, node, mask)
Пример #17
0
    def mkIndex(name, field, type, data=None):
        print "Making Index", name
        file = tmpdir + name + ".searchfile.txt"
        fi = open(file, "wb")

        for node in nodelist:
            if field == "alltext":
                s = ""
                if node.type != "directory":
                    for key, val in node.items():
                        if val:
                            s += val + "\n"
                    s += node.getName()

                # for nfile in node.getFiles():
                #    if nfile.type == "fulltext":
                #        try:
                #            fi2 = open(nfile.retrieveFile(), "rb")
                #            s += " "
                #            s += fi2.read()
                #            s += " "
                #            fi2.close()
                #        except IOError:
                #            log.error("Couldn't access file "+nfile.getName())
            elif field == "fulltext":
                s = ""
                if node.type != "directory":
                    for nfile in node.getFiles():
                        if nfile.type == "fulltext":
                            try:
                                fi2 = open(nfile.retrieveFile(), "rb")
                                s += " "
                                s += fi2.read()
                                s += " "
                                fi2.close()
                            except IOError:
                                log.error("Couldn't access file " + nfile.getName())

                #c1 = "abcdefghijklmnop"[(int(node.id)>>28)&15]
                #c2 = "abcdefghijklmnop"[(int(node.id)>>24)&15]
                #c3 = "abcdefghijklmnop"[(int(node.id)>>20)&15]
                #c4 = "abcdefghijklmnop"[(int(node.id)>>16)&15]
                #c5 = "abcdefghijklmnop"[(int(node.id)>>12)&15]
                #c6 = "abcdefghijklmnop"[(int(node.id)>>8)&15]
                #c7 = "abcdefghijklmnop"[(int(node.id)>>4)&15]
                #c8 = "abcdefghijklmnop"[(int(node.id)>>0)&15]
                #s += " mx" + c8+c7+c6+c5+c4+c3+c2+c1 + " "
            elif field == "everything":
                s = ""
                if node.type != "directory":
                    for key, val in node.items():
                        if val:
                            s += UTF8ToLatin(val) + "\n"
                    s += UTF8ToLatin(node.getName())
                    for nfile in node.getFiles():
                        if nfile.type == "fulltext":
                            try:
                                fi2 = open(nfile.retrieveFile(), "rb")
                                s += " "
                                s += fi2.read()
                                s += " "
                                fi2.close()
                            except IOError:
                                log.error("Couldn't access file " + nfile.getName())

            elif field == "objtype":
                s = node.type
                if "/" in s:
                    s = s[:s.index("/")]
            elif field == "schema":
                s = node.type
                if "/" in s:
                    s = s[s.index("/") + 1:]
                else:
                    s = ""
            elif type == "union":
                s = ""
                for item in data:
                    field = node.get(item)
                    if field:
                        s += field + " "
            else:
                s = node.get(field)
                if type == "date":
                    if len(s):
                        try:
                            s = str(date.parse_date(s).daynum())
                        except:
                            print "Couldn't parse date", s
                            s = "0"

            s = UTF8ToLatin(s)
            s = modify_tex(s, 'strip')
            fi.write(s.replace("\2", " ") + "\n")
            fi.write("\2")

        if type != "num" and type != "date":
            fi.write(
                "ecjfadf;jwkljer;jfklajd;gyugi;wyuogsdfjg;wuriosygh;nmwert;bwweriwoue;jfkajsdf;nmweurwu;hkethre;ghbyxuidfg;ewrioafi;ewirjglsag;vhxyseoru;vnmwerwe;fajsdfwetrh")
        fi.close()

        basename = name

        itype = "makeindex"
        if type == "date" or type == "num":
            itype = "makeNumIndex"
        elif type == "list" or type == "mlist" or type == "ilist":
            itype = "makeClassIndex"

        command = "%s %s %s %s %s %s" % (sys.executable, os.path.join(
            config.basedir, "core/search/runindexer.py"), itype, file, tmpdir, basename)
        exit_status = os.system(command)
        if exit_status:
            print "Exit status " + str(exit_status) + " of subprocess " + command
            sys.exit(1)