def archive_thread(self):
if not time:
return
while True:
time.sleep(int(config.get("archive.interval", 60)))
archive_nodes_3 = db.getNodeIdByAttribute("archive_state", "3")
archive_nodes_2 = []
date_now = format_date(now(), "yyymmddhhmmss")
for manager in self.manager:
# search for nodes to archive after access over period (state 2)
for n in db.getNodeIdByAttribute("archive_state", "2"):
try:
node = tree.getNode(n)
if node.get("archive_date"):
date_archive = format_date(parse_date(node.get("archive_date"), "%Y-%m-%dT%H:%M:%S"), "yyymmddhhmmss")
if date_now >= date_archive:
archive_nodes_2.append(long(node.id))
except:
pass
# union to get all nodes with state 3 and 2 with over period
archive_nodes = union((archive_nodes_3, archive_nodes_2))
nodes = intersection((db.getNodeIdByAttribute("archive_type", str(manager)), archive_nodes))
# run action defined in manager
try:
self.manager[manager].actionArchive(nodes)
except:
pass
def logs_divergence(baseline_logs, updated_logs):
log_intersection = len(intersection(baseline_logs, updated_logs))
log_union = len(union(baseline_logs, updated_logs))
IOU = 1.0
if log_union > 0:
IOU = log_intersection / float(log_union)
return 1 - IOU
def run_search(self, field, op, value):
def getSQL(type, value, spc={}): # deliver sql for given type
value = normalize_utf8(protect(u(value)))
if type == "full": # all metadata
return 'select distinct(id) from fullsearchmeta where fullsearchmeta match \'value:' + value + '\' and type <>\'directory\''
elif type == "fulltext": # fulltext
return 'select distinct(id) from textsearchmeta where textsearchmeta match \'value:' + value + '\' and type<>\'directory\''
elif type == "schema": # schemadef
return 'select distinct(id) from fullsearchmeta where schema="' + value.replace(
"'", "") + '"'
elif type == "objtype": # object type
return 'select distinct(id) from fullsearchmeta where type="' + value.replace(
"'", "") + '"'
elif type == "updatetime": # update time with operator <|>|=
if len(value) == 10:
value += "T00:00:00"
return 'select distinct(id) from searchmeta where updatetime ' + spc[
'op'] + ' "' + value.replace("t", "T") + '"'
elif type == "field":
return 'select position, name from searchmeta_def where attrname=\'' + value + '\''
elif type == "spcompare":
return 'select distinct(id) from searchmeta where schema="' + \
str(spc['pos'][1]) + '" and field' + str(spc['pos'][0]) + ' ' + spc['op'] + ' "' + value + '"'
elif type == "spfield":
return 'select distinct(id) from searchmeta where field' + str(
spc['pos'][0]) + '=""'
elif type == "spmatch":
return 'select distinct(id) from searchmeta where schema=\'' + \
str(spc['pos'][1]) + '\' and field' + str(spc['pos'][0]) + ' match \'' + value + '\''
elif type == "content_full":
return 'select * from fullsearchmeta where id=\'' + value + '\''
elif type == "content_text":
return 'select * from textsearchmeta where id=\'' + value + '\''
elif type == "content_ext":
return 'select * from searchmeta where id=\'' + value + '\''
ret = []
if value == "" or field == "" or op == "":
return []
for schema in self.schemas:
if field == "full": # all metadata incl. fulltext
res1 = self.execute(getSQL("full", value), schema,
'full') # all metadata
res2 = self.execute(getSQL("fulltext", value), schema,
'text') # fulltext
ret += union([[str(s[0]) for s in res1],
[str(s[0]) for s in res2]])
elif field == "fulltext": # fulltext
ret += [
str(s[0]) for s in self.execute(getSQL("fulltext", value),
schema, 'text')
]
elif field == "allmetadata": # all metadata
ret += [
str(s[0]) for s in self.execute(getSQL("full", value),
schema, 'full')
]
elif field == "schema":
ret += [
str(s[0]) for s in self.execute(getSQL("schema", value),
schema, 'full')
]
elif field == "objtype":
ret += [
str(s[0]) for s in self.execute(getSQL("objtype", value),
schema, 'full')
]
elif field == "updatetime":
ret += [
str(s[0]) for s in self.execute(
getSQL("updatetime", value, spc={'op': op}), schema,
'full')
]
elif field == "searchcontent":
ret = [[], [], []]
for item in self.execute(getSQL("content_full", value), schema,
'full'): # value = id
ret[0] += [i for i in item if i]
for item in self.execute(getSQL("content_ext", value), schema,
'ext'):
ret[1] += [i for i in item if i]
for item in self.execute(getSQL("content_text", value), schema,
'text'): # value = id
ret[2] += [i for i in item if i]
ret += ret
else: # special search
for pos in self.execute(getSQL("field", field), schema,
self.connames[DBTYPE]['ext']):
if op in [">=", "<="]:
res = self.execute(
getSQL("spcompare",
value,
spc={
'op': op,
'pos': pos
}), schema, 'ext')
else:
if value == "''":
res = self.execute(
getSQL("spfield", '', spc={'pos': pos}),
schema, 'ext')
else:
res = self.execute(
getSQL("spmatch", value, spc={'pos': pos}),
schema, 'ext')
res = [str(s[0]) for s in res]
if len(ret) == 0:
ret += res
if len(res) > 0:
ret += union([ret, res])
return ret
def execute(self):
ids1 = self.a.execute()
ids2 = self.b.execute()
return union([ids1, ids2])
def run_search(self, field, op, value):
def getSQL(type, value, spc={}): # deliver sql for given type
value = normalize_utf8(protect(u(value)))
if type == "full": # all metadata
return 'select distinct(id) from fullsearchmeta where fullsearchmeta match \'value:' + value + '\' and type <>\'directory\''
elif type == "fulltext": # fulltext
return 'select distinct(id) from textsearchmeta where textsearchmeta match \'value:' + value + '\' and type<>\'directory\''
elif type == "schema": # schemadef
return 'select distinct(id) from fullsearchmeta where schema="' + value.replace("'", "") + '"'
elif type == "objtype": # object type
return 'select distinct(id) from fullsearchmeta where type="' + value.replace("'", "") + '"'
elif type == "updatetime": # update time with operator <|>|=
if len(value) == 10:
value += "T00:00:00"
return 'select distinct(id) from searchmeta where updatetime ' + spc['op'] + ' "' + value.replace("t", "T") + '"'
elif type == "field":
return 'select position, name from searchmeta_def where attrname=\'' + value + '\''
elif type == "spcompare":
return 'select distinct(id) from searchmeta where schema="' + \
str(spc['pos'][1]) + '" and field' + str(spc['pos'][0]) + ' ' + spc['op'] + ' "' + value + '"'
elif type == "spfield":
return 'select distinct(id) from searchmeta where field' + str(spc['pos'][0]) + '=""'
elif type == "spmatch":
return 'select distinct(id) from searchmeta where schema=\'' + \
str(spc['pos'][1]) + '\' and field' + str(spc['pos'][0]) + ' match \'' + value + '\''
elif type == "content_full":
return 'select * from fullsearchmeta where id=\'' + value + '\''
elif type == "content_text":
return 'select * from textsearchmeta where id=\'' + value + '\''
elif type == "content_ext":
return 'select * from searchmeta where id=\'' + value + '\''
ret = []
if value == "" or field == "" or op == "":
return []
for schema in self.schemas:
if field == "full": # all metadata incl. fulltext
res1 = self.execute(getSQL("full", value), schema, 'full') # all metadata
res2 = self.execute(getSQL("fulltext", value), schema, 'text') # fulltext
ret += union([[str(s[0]) for s in res1], [str(s[0]) for s in res2]])
elif field == "fulltext": # fulltext
ret += [str(s[0]) for s in self.execute(getSQL("fulltext", value),
schema,
'text')]
elif field == "allmetadata": # all metadata
ret += [str(s[0]) for s in self.execute(getSQL("full", value),
schema,
'full')]
elif field == "schema":
ret += [str(s[0]) for s in self.execute(getSQL("schema", value),
schema,
'full')]
elif field == "objtype":
ret += [str(s[0]) for s in self.execute(getSQL("objtype", value),
schema,
'full')]
elif field == "updatetime":
ret += [str(s[0]) for s in self.execute(getSQL("updatetime", value, spc={'op': op}),
schema,
'full')]
elif field == "searchcontent":
ret = [[], [], []]
for item in self.execute(getSQL("content_full", value), schema, 'full'): # value = id
ret[0] += [i for i in item if i]
for item in self.execute(getSQL("content_ext", value), schema, 'ext'):
ret[1] += [i for i in item if i]
for item in self.execute(getSQL("content_text", value), schema, 'text'): # value = id
ret[2] += [i for i in item if i]
ret += ret
else: # special search
for pos in self.execute(getSQL("field", field), schema, self.connames[DBTYPE]['ext']):
if op in [">=", "<="]:
res = self.execute(getSQL("spcompare", value, spc={'op': op, 'pos': pos}), schema, 'ext')
else:
if value == "''":
res = self.execute(getSQL("spfield", '', spc={'pos': pos}), schema, 'ext')
else:
res = self.execute(getSQL("spmatch", value, spc={'pos': pos}), schema, 'ext')
res = [str(s[0]) for s in res]
if len(ret) == 0:
ret += res
if len(res) > 0:
ret += union([ret, res])
return ret
def execute(self):
return union([self.a.execute(), self.b.execute()])
def execute(self):
return union([self.a.execute(), self.b.execute()])
def execute(self):
ids1 = self.a.execute()
ids2 = self.b.execute()
return union([ids1, ids2])