def getFormattedValue(self,
metafield,
maskitem,
mask,
node,
language,
html=True,
template_from_caller=None):
value = node.get_special(metafield.name)
# consider int, long values like filesize
if isinstance(value, (int, long)):
value = str(value)
value = value.replace(";", "; ")
# ignore trailing newlines for textfields
value = value.rstrip("\r\n")
if value.find('\n') != -1:
valuesList = value.split('\n')
if any(lang in valuesList
for lang in config.languages): # treat as multilingual
index = 0
try:
index = valuesList.index(language)
value = valuesList[index + 1]
except ValueError as e:
msg = "Exception in getFormattedValue for textfield:\n"
msg += " valuesList=%r\n" % valuesList
msg += " node.name=%r, node.id=%r, node.type=%r\n" % (
node.name, node.id, node.type)
msg += " metafield.name=%r, metafield.id=%r, metafield.type=%r\n" % (
metafield.name, metafield.id, metafield.type)
msg += " language=%r, mask=%r" % (language, mask)
logg.exception(msg)
value = u""
else:
# treat as monolingual
pass
if html:
value = esc(value)
# replace variables
# substitute TeX sub/super-scripts with <sub>/<sup> html tags
value = modify_tex(value, 'html')
value = replace_attribute_variables(value, node.id, node.get,
r'<(.+?)>', "<", ">")
value = value.replace("<", "<").replace(">", ">")
if not maskitem:
return (metafield.getLabel(), value)
# use default value from mask if value is empty
if value == u'':
value = maskitem.getDefault()
return (metafield.getLabel(), value)
def getViewHTML(self, fields, nodes, flags, language="", template_from_caller=None, mask=None):
ret = ""
node = nodes[0]
mask = fields[0].parents.first()
separator = ""
if mask.getMappingHeader() != "":
ret += mask.getMappingHeader() + "\r\n"
field_vals = []
for field in fields:
attribute_nid = field.get("attribute", None)
if attribute_nid is None:
continue
try:
attribute_nid = int(attribute_nid)
except ValueError:
logg.warn("ignoring field # %s with invalid attribute id: '%r'", field.id, attribute_nid)
continue
attrnode = q(Node).get(attribute_nid)
if attrnode is None:
continue
if field.get("fieldtype") == "mapping": # mapping to mapping definition
exportmapping_id = mask.get("exportmapping").split(";")[0]
mapping = q(Node).get(exportmapping_id)
if mapping is None:
logg.warn("exportmapping %s for mask %s not found", exportmapping_id, mask.id)
return u""
separator = mapping.get("separator")
ns = mapping.getNamespace()
if ns != "":
ns += ":"
fld = q(Node).get(field.get("mappingfield"))
format = fld.getExportFormat()
field_value = ns + fld.getName()
default = fld.getDefault().strip()
else: # attributes of node
format = field.get("mappingfield")
field_value = ""
default = ""
field_vals.append(self.replaceVars(format, node, attrnode, field_value, options=mask.getExportOptions(), mask=mask, default=default))
if not mask.hasExportOption("l"):
ret += separator.join(field_vals)
else:
ret += u"".join(field_vals)
if mask.getMappingFooter() != "":
ret += "\r\n" + mask.getMappingFooter()
ret = modify_tex(ret, 'strip')
return self.replaceVars(ret, node, mask)
def getViewHTML(self,
fields,
nodes,
flags,
language="",
template_from_caller=None,
mask=None):
ret = ""
node = nodes[0]
mask = fields[0].getParents()[0]
separator = ""
if mask.getMappingHeader() != "":
ret += mask.getMappingHeader() + "\r\n"
for field in fields:
try:
attrnode = tree.getNode(field.get("attribute"))
except:
continue
if field.get(
"fieldtype") == "mapping": # mapping to mapping definition
mapping = tree.getNode(mask.get("exportmapping").split(";")[0])
separator = mapping.get("separator")
ns = mapping.getNamespace()
if ns != "":
ns += ":"
fld = tree.getNode(field.get("mappingfield"))
format = fld.getExportFormat()
field_value = ns + fld.getName()
default = fld.getDefault().strip()
else: # attributes of node
format = field.get("mappingfield")
field_value = ""
default = ""
ret += self.replaceVars(format,
node,
attrnode,
field_value,
options=mask.getExportOptions(),
mask=mask,
default=default)
if not mask.hasExportOption(
"l") and list(fields).index(field) < len(fields) - 1:
ret += separator
if mask.getMappingFooter() != "":
ret += "\r\n" + mask.getMappingFooter()
ret = modify_tex(ret, 'strip')
return self.replaceVars(ret, node, mask)
def nodeToExtSearch(self, node,
schema): # build extended search index from node
if len(node.getSearchFields()
) == 0: # stop if schema has no searchfields
return True
self.nodeToSchemaDef(node, schema) # save definition
keyvalue = []
i = 1
for field in node.getSearchFields():
key = "field%d" % i
i += 1
value = ""
if field.getFieldtype() == "union":
for item in field.get("valuelist").split(";"):
value += node.get(item) + '|'
else:
value = node.get(field.getName())
keyvalue += [(key, modify_tex(u(protect(value)), 'strip'))]
sql0 = 'SELECT id FROM searchmeta where id=\'{}\''.format(node.id)
sql1 = 'UPDATE searchmeta SET '
sql2 = 'INSERT INTO searchmeta (id, type, schema, updatetime'
for key, value in keyvalue:
sql1 += key + "='" + normalize_utf8(value) + "', "
sql2 += ", "
sql2 += key
sql1 += "type='" + node.getContentType(
) + "', schema='" + schema + "', updatetime='" + node.get(
"updatetime") + "'"
sql2 += ") VALUES("
sql2 += '\'{}\', "{}", "{}", "{}"'.format(node.id,
node.getContentType(),
schema,
node.get("updatetime"))
for key, value in keyvalue:
sql2 += ", '" + normalize_utf8(value) + "'"
sql1 += " WHERE id='{}'".format(node.id)
sql2 += ")"
sql = ""
try:
sql = sql0
if self.execute(sql0, schema, 'ext'): # select
sql = sql1
self.execute(sql1, schema, 'ext') # do update
else:
sql = sql2
self.execute(sql2, schema, 'ext') # do insert
return True
except:
logException('error in sqlite insert/update: ' + sql)
return False
def nodeToExtSearch(self, node, schema): # build extended search index from node
if len(node.getSearchFields()) == 0: # stop if schema has no searchfields
return True
self.nodeToSchemaDef(node, schema) # save definition
keyvalue = []
i = 1
for field in node.getSearchFields():
key = "field%d" % i
i += 1
value = ""
if field.getFieldtype() == "union":
for item in field.get("valuelist").split(";"):
value += node.get(item) + '|'
else:
value = node.get(field.getName())
keyvalue += [(key, modify_tex(u(protect(value)), 'strip'))]
sql0 = 'SELECT id FROM searchmeta where id=\'{}\''.format(node.id)
sql1 = 'UPDATE searchmeta SET '
sql2 = 'INSERT INTO searchmeta (id, type, schema, updatetime'
for key, value in keyvalue:
sql1 += key + "='" + normalize_utf8(value) + "', "
sql2 += ", "
sql2 += key
sql1 += "type='" + node.getContentType() + "', schema='" + schema + "', updatetime='" + node.get("updatetime") + "'"
sql2 += ") VALUES("
sql2 += '\'{}\', "{}", "{}", "{}"'.format(node.id,
node.getContentType(),
schema,
node.get("updatetime"))
for key, value in keyvalue:
sql2 += ", '" + normalize_utf8(value) + "'"
sql1 += " WHERE id='{}'".format(node.id)
sql2 += ")"
sql = ""
try:
sql = sql0
if self.execute(sql0, schema, 'ext'): # select
sql = sql1
self.execute(sql1, schema, 'ext') # do update
else:
sql = sql2
self.execute(sql2, schema, 'ext') # do insert
return True
except:
logException('error in sqlite insert/update: ' + sql)
return False
def getFormattedValue(self, metafield, maskitem, mask, node, language, html=True, template_from_caller=None):
value = node.get_special(metafield.name)
# consider int, long values like filesize
if isinstance(value, (int, long)):
value = str(value)
value = value.replace(";", "; ")
# ignore trailing newlines for textfields
value = value.rstrip("\r\n")
if value.find('\n') != -1:
valuesList = value.split('\n')
if any(lang in valuesList for lang in config.languages): # treat as multilingual
index = 0
try:
index = valuesList.index(language)
value = valuesList[index + 1]
except ValueError as e:
msg = "Exception in getFormattedValue for textfield:\n"
msg += " valuesList=%r\n" % valuesList
msg += " node.name=%r, node.id=%r, node.type=%r\n" % (node.name, node.id, node.type)
msg += " metafield.name=%r, metafield.id=%r, metafield.type=%r\n" % (metafield.name, metafield.id, metafield.type)
msg += " language=%r, mask=%r" % (language, mask)
logg.exception(msg)
value = u""
else:
# treat as monolingual
pass
if html:
value = esc(value)
# replace variables
# substitute TeX sub/super-scripts with <sub>/<sup> html tags
value = modify_tex(value, 'html')
value = replace_attribute_variables(value, node.id, node.get, r'<(.+?)>', "<", ">")
value = value.replace("<", "<").replace(">", ">")
if not maskitem:
return (metafield.getLabel(), value)
# use default value from mask if value is empty
if value == u'':
value = maskitem.getDefault()
return (metafield.getLabel(), value)
def nodeToSimpleSearch(self,
node,
schema,
type=""): # build simple search index from node
sql_upd = "UPDATE fullsearchmeta SET type='{}', schema='{}', value='{}| ".format(
node.getContentType(), node.getSchema(), node.name)
sql_ins = "INSERT INTO fullsearchmeta (id, type, schema, value) VALUES('{}', '{}', '{}', '{}| ".format(
node.id, node.getContentType(), node.getSchema(), node.name)
# attributes
val = ''
for key, value in node.items():
if key not in SYSTEMATTRS: # ignore system attributes
val += protect(u(value)) + '| '
for v in val.split(" "):
v = u(v)
if normalize_utf8(v) != v.lower():
val += ' ' + normalize_utf8(v)
val = val.replace(chr(0), "") + ' '
# remove tex markup
val = modify_tex(val, 'strip')
# files
for file in node.getFiles():
val += protect(
u(file.getName() + '| ' + file.getType() + '| ' +
file.getMimeType()) + '| ')
sql_upd += val + '\' WHERE id=\'{}\''.format(node.id)
sql_ins += val + '\')'
sql = ""
try:
sql = 'SELECT id from fullsearchmeta WHERE id=\'{}\''.format(
node.id)
if self.execute(sql, schema, 'full'): # check existance
sql = sql_upd # do update
else:
sql = sql_ins # do insert
self.execute(sql, schema, 'full')
return True
except:
logException('error in sqlite insert/update: ' + sql)
return False
def getViewHTML(self, fields, nodes, flags, language="", template_from_caller=None, mask=None):
ret = ""
node = nodes[0]
mask = fields[0].getParents()[0]
separator = ""
if mask.getMappingHeader() != "":
ret += mask.getMappingHeader() + "\r\n"
for field in fields:
try:
attrnode = tree.getNode(field.get("attribute"))
except:
continue
if field.get("fieldtype") == "mapping": # mapping to mapping definition
mapping = tree.getNode(mask.get("exportmapping").split(";")[0])
separator = mapping.get("separator")
ns = mapping.getNamespace()
if ns != "":
ns += ":"
fld = tree.getNode(field.get("mappingfield"))
format = fld.getExportFormat()
field_value = ns + fld.getName()
default = fld.getDefault().strip()
else: # attributes of node
format = field.get("mappingfield")
field_value = ""
default = ""
ret += self.replaceVars(
format, node, attrnode, field_value, options=mask.getExportOptions(), mask=mask, default=default
)
if not mask.hasExportOption("l") and list(fields).index(field) < len(fields) - 1:
ret += separator
if mask.getMappingFooter() != "":
ret += "\r\n" + mask.getMappingFooter()
ret = modify_tex(ret, "strip")
return self.replaceVars(ret, node, mask)
def nodeToSimpleSearch(self, node, schema, type=""): # build simple search index from node
sql_upd ="UPDATE fullsearchmeta SET type='{}', schema='{}', value='{}| ".format(node.getContentType(),
node.getSchema(),
node.name)
sql_ins = "INSERT INTO fullsearchmeta (id, type, schema, value) VALUES('{}', '{}', '{}', '{}| ".format(node.id,
node.getContentType(),
node.getSchema(),
node.name)
# attributes
val = ''
for key, value in node.items():
if key not in SYSTEMATTRS: # ignore system attributes
val += protect(u(value)) + '| '
for v in val.split(" "):
v = u(v)
if normalize_utf8(v) != v.lower():
val += ' ' + normalize_utf8(v)
val = val.replace(chr(0), "") + ' '
# remove tex markup
val = modify_tex(val, 'strip')
# files
for file in node.getFiles():
val += protect(u(file.getName() + '| ' + file.getType() + '| ' + file.getMimeType()) + '| ')
sql_upd += val + '\' WHERE id=\'{}\''.format(node.id)
sql_ins += val + '\')'
sql = ""
try:
sql = 'SELECT id from fullsearchmeta WHERE id=\'{}\''.format(node.id)
if self.execute(sql, schema, 'full'): # check existance
sql = sql_upd # do update
else:
sql = sql_ins # do insert
self.execute(sql, schema, 'full')
return True
except:
logException('error in sqlite insert/update: ' + sql)
return False
def get_extended_field_ratio(schema, node, db_content):
"""
Compares the values in the ext search db and the values in the node instance and returns
a ratio of likeness between the two values.
@param schema: String, name of the schema
@param node: Node, an core.tree node instance
@return: Float
"""
ratios = []
field_names = get_zero_index_schema_fields(schema)
for field in field_names:
node_value = normalize_utf8(modify_tex(u(protect(node.get(field.name))), 'strip'))
db_value = str(db_content[field.position])
equality_ratio = difflib.SequenceMatcher(None, db_value, node_value).ratio()
ratios.append(equality_ratio)
return sum(ratios) / len(ratios)
def get_extended_field_ratio(schema, node, db_content):
"""
Compares the values in the ext search db and the values in the node instance and returns
a ratio of likeness between the two values.
@param schema: String, name of the schema
@param node: Node, an core.tree node instance
@return: Float
"""
ratios = []
field_names = get_zero_index_schema_fields(schema)
for field in field_names:
node_value = normalize_utf8(
modify_tex(u(protect(node.get(field.name))), 'strip'))
db_value = str(db_content[field.position])
equality_ratio = difflib.SequenceMatcher(None, db_value,
node_value).ratio()
ratios.append(equality_ratio)
return sum(ratios) / len(ratios)
def getFormatedValue(self,
field,
node,
language=None,
html=1,
template_from_caller=None,
mask=None):
value = node.get(field.getName()).replace(";", "; ")
# ignore trailing newlines for textfields
value = value.rstrip("\r\n")
if value.find('\n') != -1:
valuesList = value.split('\n')
if any(lang in valuesList
for lang in system_languages): # treat as multilingual
index = 0
try:
index = valuesList.index(language)
value = valuesList[index + 1]
except ValueError as e:
logException(e)
log = logging.getLogger("errors")
msg = "Exception in getFormatedValue for textfield:\n"
msg += " valuesList=%r\n" % valuesList
msg += " node.name=%r, node.id=%r, node.type=%r\n" % (
node.name, node.id, node.type)
msg += " field.name=%r, field.id=%r, field.type=%r\n" % (
field.name, field.id, field.type)
msg += " language=%r, mask=%r" % (language, mask)
log.error(msg)
value = ""
else:
# treat as monolingual
pass
unescaped_value = value
if html:
value = esc(value)
# replace variables
# substitute TeX sub/super-scripts with <sub>/<sup> html tags
value = modify_tex(value, 'html')
for var in re.findall(r'<(.+?)>', value):
if var == "att:id":
value = value.replace("<" + var + ">", node.id)
elif var.startswith("att:"):
val = node.get(var[4:])
if val == "":
val = "____"
value = value.replace("<" + var + ">", val)
value = value.replace("<", "<").replace(">", ">")
maskitem = getMaskitemForField(field, language=language, mask=mask)
if not maskitem:
return (field.getLabel(), value)
# use default value from mask if value is empty
if value == '':
value = maskitem.getDefault()
if template_from_caller and template_from_caller[
0] and maskitem and str(
maskitem.id) == template_from_caller[3]:
value = template_from_caller[0]
context = {'node': node, 'host': "http://" + config.get("host.name")}
if (template_from_caller and
template_from_caller[0]) and (not node.get(field.getName())):
value = runTALSnippet(value, context)
else:
try:
value = runTALSnippet(value, context)
except:
value = runTALSnippet(unescaped_value, context)
return (field.getLabel(), value)
def mkIndex(name, field, type, data=None):
print "Making Index", name
file = tmpdir + name + ".searchfile.txt"
fi = open(file, "wb")
for node in nodelist:
if field == "alltext":
s = ""
if node.type != "directory":
for key, val in node.items():
if val:
s += val + "\n"
s += node.getName()
# for nfile in node.getFiles():
# if nfile.type == "fulltext":
# try:
# fi2 = open(nfile.retrieveFile(), "rb")
# s += " "
# s += fi2.read()
# s += " "
# fi2.close()
# except IOError:
# log.error("Couldn't access file "+nfile.getName())
elif field == "fulltext":
s = ""
if node.type != "directory":
for nfile in node.getFiles():
if nfile.type == "fulltext":
try:
fi2 = open(nfile.retrieveFile(), "rb")
s += " "
s += fi2.read()
s += " "
fi2.close()
except IOError:
log.error("Couldn't access file " +
nfile.getName())
#c1 = "abcdefghijklmnop"[(int(node.id)>>28)&15]
#c2 = "abcdefghijklmnop"[(int(node.id)>>24)&15]
#c3 = "abcdefghijklmnop"[(int(node.id)>>20)&15]
#c4 = "abcdefghijklmnop"[(int(node.id)>>16)&15]
#c5 = "abcdefghijklmnop"[(int(node.id)>>12)&15]
#c6 = "abcdefghijklmnop"[(int(node.id)>>8)&15]
#c7 = "abcdefghijklmnop"[(int(node.id)>>4)&15]
#c8 = "abcdefghijklmnop"[(int(node.id)>>0)&15]
#s += " mx" + c8+c7+c6+c5+c4+c3+c2+c1 + " "
elif field == "everything":
s = ""
if node.type != "directory":
for key, val in node.items():
if val:
s += UTF8ToLatin(val) + "\n"
s += UTF8ToLatin(node.getName())
for nfile in node.getFiles():
if nfile.type == "fulltext":
try:
fi2 = open(nfile.retrieveFile(), "rb")
s += " "
s += fi2.read()
s += " "
fi2.close()
except IOError:
log.error("Couldn't access file " +
nfile.getName())
elif field == "objtype":
s = node.type
if "/" in s:
s = s[:s.index("/")]
elif field == "schema":
s = node.type
if "/" in s:
s = s[s.index("/") + 1:]
else:
s = ""
elif type == "union":
s = ""
for item in data:
field = node.get(item)
if field:
s += field + " "
else:
s = node.get(field)
if type == "date":
if len(s):
try:
s = str(date.parse_date(s).daynum())
except:
print "Couldn't parse date", s
s = "0"
s = UTF8ToLatin(s)
s = modify_tex(s, 'strip')
fi.write(s.replace("\2", " ") + "\n")
fi.write("\2")
if type != "num" and type != "date":
fi.write(
"ecjfadf;jwkljer;jfklajd;gyugi;wyuogsdfjg;wuriosygh;nmwert;bwweriwoue;jfkajsdf;nmweurwu;hkethre;ghbyxuidfg;ewrioafi;ewirjglsag;vhxyseoru;vnmwerwe;fajsdfwetrh"
)
fi.close()
basename = name
itype = "makeindex"
if type == "date" or type == "num":
itype = "makeNumIndex"
elif type == "list" or type == "mlist" or type == "ilist":
itype = "makeClassIndex"
command = "%s %s %s %s %s %s" % (
sys.executable,
os.path.join(config.basedir, "core/search/runindexer.py"), itype,
file, tmpdir, basename)
exit_status = os.system(command)
if exit_status:
print "Exit status " + str(
exit_status) + " of subprocess " + command
sys.exit(1)
def getFormatedValue(self, field, node, language=None, html=1, template_from_caller=None, mask=None):
value = node.get(field.getName()).replace(";", "; ")
# ignore trailing newlines for textfields
value = value.rstrip("\r\n")
if value.find('\n') != -1:
valuesList = value.split('\n')
if any(lang in valuesList for lang in system_languages): # treat as multilingual
index = 0
try:
index = valuesList.index(language)
value = valuesList[index + 1]
except ValueError as e:
logException(e)
log = logging.getLogger("errors")
msg = "Exception in getFormatedValue for textfield:\n"
msg += " valuesList=%r\n" % valuesList
msg += " node.name=%r, node.id=%r, node.type=%r\n" % (node.name, node.id, node.type)
msg += " field.name=%r, field.id=%r, field.type=%r\n" % (field.name, field.id, field.type)
msg += " language=%r, mask=%r" % (language, mask)
log.error(msg)
value = ""
else:
# treat as monolingual
pass
unescaped_value = value
if html:
value = esc(value)
# replace variables
# substitute TeX sub/super-scripts with <sub>/<sup> html tags
value = modify_tex(value, 'html')
for var in re.findall(r'<(.+?)>', value):
if var == "att:id":
value = value.replace("<" + var + ">", node.id)
elif var.startswith("att:"):
val = node.get(var[4:])
if val == "":
val = "____"
value = value.replace("<" + var + ">", val)
value = value.replace("<", "<").replace(">", ">")
maskitem = getMaskitemForField(field, language=language, mask=mask)
if not maskitem:
return (field.getLabel(), value)
# use default value from mask if value is empty
if value == '':
value = maskitem.getDefault()
if template_from_caller and template_from_caller[0] and maskitem and str(maskitem.id) == template_from_caller[3]:
value = template_from_caller[0]
context = {'node': node, 'host': "http://" + config.get("host.name")}
if (template_from_caller and template_from_caller[0]) and (not node.get(field.getName())):
value = runTALSnippet(value, context)
else:
try:
value = runTALSnippet(value, context)
except:
value = runTALSnippet(unescaped_value, context)
return (field.getLabel(), value)
def write_formatted_response(req,
path,
params,
data,
id,
debug=True,
allchildren=False,
singlenode=False,
parents=False):
atime = starttime = time.time()
r_timetable = []
_p = params.copy()
if "jsoncallback" in _p:
del _p['jsoncallback']
if "_" in _p:
del _p['_']
send_children = "send_children" in params
cache_key = '|'.join(
map(str, [path, _p, allchildren, singlenode, parents, send_children]))
cache_key = cache_key.replace(' ', '_')
acceptcached = float(params.get('acceptcached', DEFAULT_CACHE_VALID))
result_from_cache = None
if acceptcached > 0.0:
resultcode, cachecontent = resultcache.retrieve(
cache_key, acceptcached)
if resultcode == 'hit':
cache_name = 'resultcache'
timestamp_from_cache = time_cached = resultcache.getTimestamp(
cache_key)
time_delta = starttime - time_cached
result_from_cache, mimetype_from_cache = cachecontent[-1]
# replace jQuery, jsonp callback value
if result_from_cache.startswith(
'jQuery') or result_from_cache.startswith('jsonp'):
result_from_cache = params['jsoncallback'] + result_from_cache[
result_from_cache.find("({"):]
r_timetable.append([
"retrieved filtered result from 'resultcache': (%d bytes), time_delta: %.3f lower acceptcached %.3f sec."
% (len(result_from_cache), time_delta, acceptcached),
time.time() - atime
])
atime = time.time()
elif resultcode == 'refused':
time_cached = resultcache.getTimestamp(cache_key)
time_delta = starttime - time_cached
r_timetable.append([
"cached result exists in 'resultcache', but not used: time_delta: %.3f sec. higher acceptcached %.3f sec."
% (time_delta, acceptcached),
time.time() - atime
])
atime = time.time()
if not result_from_cache:
res_format = (params.get('format', 'xml')).lower()
# XXX: hack because we want all files for the XML format only
if res_format == "xml":
fetch_files = True
else:
fetch_files = False
d = get_node_data_struct(req,
path,
params,
data,
id,
debug=debug,
allchildren=allchildren,
singlenode=singlenode,
send_children=send_children,
parents=parents,
fetch_files=fetch_files,
csv=res_format == u'csv')
if r_timetable:
d['timetable'] = r_timetable + d.setdefault('timetable', [])
r_timetable = []
formatIsSupported = False
for supported_format in supported_formats:
if res_format in supported_format[0]:
atime = time.time()
formatIsSupported = True
s = supported_format[1](req,
path,
params,
data,
d,
debug=debug,
singlenode=singlenode,
send_children=send_children)
if res_format == 'json' and 'jsoncallback' in params:
s = params['jsoncallback'] + '(' + s + ')'
# the return value of this kind of call must be interpreted as javascript,
# so we must set the mimetype or browsers will complain
mimetype = "application/javascript"
else:
# XXX: clients can override the content_type by setting the mimetype param
# XXX: this is ugly, but we keep it for compatibility
mimetype = params.get('mimetype', supported_format[2])
# append correct charset if client didn't force another value
# it doesn't make sense to set it in the client to a different charset than utf8, but it was possible in the past...
if "charset=" in mimetype:
content_type = mimetype
else:
content_type = mimetype + "; charset=utf-8"
d['timetable'].append(
["formatted for '%s'" % res_format,
time.time() - atime])
atime = time.time()
disposition = params.get('disposition', '')
if disposition:
# ex.: (direct to download) value: "attachment; filename=myfilename.txt"
# ex.: (open in browser) value: "filename=myfilename.txt"
req.reply_headers['Content-Disposition'] = disposition
d['timetable'].append([
"wrote disposition %r to reply header" % (disposition),
time.time() - atime
])
atime = time.time()
break
if not formatIsSupported:
d['status'] = 'fail'
d['html_response_code'] = '404' # not found
d['errormessage'] = 'unsupported format'
d['build_response_end'] = time.time()
s = struct2xml(req,
path,
params,
data,
d,
singlenode=True,
send_children=False,
fetch_files=True)
content_type = "text/xml; charset=utf-8"
if acceptcached > 0: # only write to cache for these requests
resultcache.update(cache_key, [s, mimetype])
d['timetable'].append([
"wrote result to 'resultcache' (%d bytes), now in cache: %d entries"
% (len(s), resultcache.getKeysCount()),
time.time() - atime
])
atime = time.time()
#d['timetable'].append(["wrote result to 'resultcache' (%d bytes), now in cache: %d entries: \r\n%s" % (len(s), resultcache.getKeysCount(), resultcache.report()), time.time()-atime]); atime = time.time()
# d['timetable'].append(["wrote result to 'resultcache' (%d bytes), now in
# cache: %d entries: %s" % (len(s), resultcache.getKeysCount(),
# "#".join(resultcache.getKeys())), time.time()-atime]); atime =
# time.time()
s = modify_tex(s.decode("utf8"), 'strip').encode("utf8")
else:
d = {}
d['status'] = 'ok'
d['html_response_code'] = '200' # ok
d['build_response_end'] = time.time()
if r_timetable:
d['timetable'] = r_timetable
s = result_from_cache
d['timetable'].append([
"serving %.3f sec. old response (%d bytes) from '%s', cache_key: %s"
% (time.time() - timestamp_from_cache, len(s), cache_name,
cache_key),
time.time() - atime
])
atime = time.time()
mimetype = mimetype_from_cache
content_type = mimetype + "; charset=utf-8"
req.reply_headers['Content-Type'] = content_type
def compressForDeflate(s):
import gzip
return gzip.zlib.compress(s, 9)
def compressForGzip(s):
import cStringIO
import gzip
buffer = cStringIO.StringIO()
gzfile = gzip.GzipFile(mode='wb', fileobj=buffer, compresslevel=9)
gzfile.write(s)
gzfile.close()
return buffer.getvalue()
if 'deflate' in params:
size_uncompressed = len(s)
compressed_s = compressForDeflate(s)
s = compressed_s
size_compressed = len(s)
try:
percentage = 100.0 * size_compressed / size_uncompressed
except:
percentage = 100.0
req.reply_headers['Content-Encoding'] = "deflate"
d['timetable'].append([
"'deflate' in request: executed compressForDeflate(s), %d bytes -> %d bytes (compressed to: %.1f %%)"
% (size_uncompressed, size_compressed, percentage),
time.time() - atime
])
atime = time.time()
elif 'gzip' in params:
size_uncompressed = len(s)
compressed_s = compressForGzip(s)
s = compressed_s
size_compressed = len(s)
try:
percentage = 100.0 * size_compressed / size_uncompressed
except:
percentage = 100.0
req.reply_headers['Content-Encoding'] = "gzip"
d['timetable'].append([
"'gzip' in request: executed compressForGzip(s), %d bytes -> %d bytes (compressed to: %.1f %%)"
% (size_uncompressed, size_compressed, percentage),
time.time() - atime
])
atime = time.time()
# (format) Expires: Mon, 28 Nov 2011 12:41:22 GMT
# see core.athana.build_http_date
# req.reply_headers['Expires'] = time.strftime ('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(time.time()+60.0)) # 1 minute
# remark: on 2011-12-01 switched response from req.write to sendAsBuffer for performance reasons
# (before: ) req.write(s)
_sendAsBuffer(req,
s,
content_type,
force=1,
allow_cross_origin=allow_cross_origin)
d['timetable'].append([
"executed sendAsBuffer, %d bytes, content type='%s'" %
(len(s), content_type),
time.time() - atime
])
atime = time.time()
return d['html_response_code'], len(s), d
def getViewHTML(self,
fields,
nodes,
flags,
language="",
template_from_caller=None,
mask=None):
ret = ""
node = nodes[0]
mask = fields[0].parents.first()
separator = ""
if mask.getMappingHeader() != "":
ret += mask.getMappingHeader() + "\r\n"
field_vals = []
for field in fields:
attribute_nid = field.get("attribute", None)
if attribute_nid is None:
continue
try:
attribute_nid = int(attribute_nid)
except ValueError:
logg.warn(
"ignoring field # %s with invalid attribute id: '%r'",
field.id, attribute_nid)
continue
attrnode = q(Node).get(attribute_nid)
if attrnode is None:
continue
if field.get(
"fieldtype") == "mapping": # mapping to mapping definition
exportmapping_id = mask.get("exportmapping").split(";")[0]
mapping = q(Node).get(exportmapping_id)
if mapping is None:
logg.warn("exportmapping %s for mask %s not found",
exportmapping_id, mask.id)
return u""
separator = mapping.get("separator")
ns = mapping.getNamespace()
if ns != "":
ns += ":"
fld = q(Node).get(field.get("mappingfield"))
format = fld.getExportFormat()
field_value = ns + fld.getName()
default = fld.getDefault().strip()
else: # attributes of node
format = field.get("mappingfield")
field_value = ""
default = ""
field_vals.append(
self.replaceVars(format,
node,
attrnode,
field_value,
options=mask.getExportOptions(),
mask=mask,
default=default))
if not mask.hasExportOption("l"):
ret += separator.join(field_vals)
else:
ret += u"".join(field_vals)
if mask.getMappingFooter() != "":
ret += "\r\n" + mask.getMappingFooter()
ret = modify_tex(ret, 'strip')
return self.replaceVars(ret, node, mask)
def mkIndex(name, field, type, data=None):
print "Making Index", name
file = tmpdir + name + ".searchfile.txt"
fi = open(file, "wb")
for node in nodelist:
if field == "alltext":
s = ""
if node.type != "directory":
for key, val in node.items():
if val:
s += val + "\n"
s += node.getName()
# for nfile in node.getFiles():
# if nfile.type == "fulltext":
# try:
# fi2 = open(nfile.retrieveFile(), "rb")
# s += " "
# s += fi2.read()
# s += " "
# fi2.close()
# except IOError:
# log.error("Couldn't access file "+nfile.getName())
elif field == "fulltext":
s = ""
if node.type != "directory":
for nfile in node.getFiles():
if nfile.type == "fulltext":
try:
fi2 = open(nfile.retrieveFile(), "rb")
s += " "
s += fi2.read()
s += " "
fi2.close()
except IOError:
log.error("Couldn't access file " + nfile.getName())
#c1 = "abcdefghijklmnop"[(int(node.id)>>28)&15]
#c2 = "abcdefghijklmnop"[(int(node.id)>>24)&15]
#c3 = "abcdefghijklmnop"[(int(node.id)>>20)&15]
#c4 = "abcdefghijklmnop"[(int(node.id)>>16)&15]
#c5 = "abcdefghijklmnop"[(int(node.id)>>12)&15]
#c6 = "abcdefghijklmnop"[(int(node.id)>>8)&15]
#c7 = "abcdefghijklmnop"[(int(node.id)>>4)&15]
#c8 = "abcdefghijklmnop"[(int(node.id)>>0)&15]
#s += " mx" + c8+c7+c6+c5+c4+c3+c2+c1 + " "
elif field == "everything":
s = ""
if node.type != "directory":
for key, val in node.items():
if val:
s += UTF8ToLatin(val) + "\n"
s += UTF8ToLatin(node.getName())
for nfile in node.getFiles():
if nfile.type == "fulltext":
try:
fi2 = open(nfile.retrieveFile(), "rb")
s += " "
s += fi2.read()
s += " "
fi2.close()
except IOError:
log.error("Couldn't access file " + nfile.getName())
elif field == "objtype":
s = node.type
if "/" in s:
s = s[:s.index("/")]
elif field == "schema":
s = node.type
if "/" in s:
s = s[s.index("/") + 1:]
else:
s = ""
elif type == "union":
s = ""
for item in data:
field = node.get(item)
if field:
s += field + " "
else:
s = node.get(field)
if type == "date":
if len(s):
try:
s = str(date.parse_date(s).daynum())
except:
print "Couldn't parse date", s
s = "0"
s = UTF8ToLatin(s)
s = modify_tex(s, 'strip')
fi.write(s.replace("\2", " ") + "\n")
fi.write("\2")
if type != "num" and type != "date":
fi.write(
"ecjfadf;jwkljer;jfklajd;gyugi;wyuogsdfjg;wuriosygh;nmwert;bwweriwoue;jfkajsdf;nmweurwu;hkethre;ghbyxuidfg;ewrioafi;ewirjglsag;vhxyseoru;vnmwerwe;fajsdfwetrh")
fi.close()
basename = name
itype = "makeindex"
if type == "date" or type == "num":
itype = "makeNumIndex"
elif type == "list" or type == "mlist" or type == "ilist":
itype = "makeClassIndex"
command = "%s %s %s %s %s %s" % (sys.executable, os.path.join(
config.basedir, "core/search/runindexer.py"), itype, file, tmpdir, basename)
exit_status = os.system(command)
if exit_status:
print "Exit status " + str(exit_status) + " of subprocess " + command
sys.exit(1)