def getStatFile(node, timestamp, type, period=period):
f = None
for file in node.getFiles():
if file.getType() == "statistic":
try:
if file.getName() == "stat_{}_{}_{}.xml".format(node.id, timestamp, type):
if timestamp == str(format_date(now(), "yyyy-mm")) or timestamp == period: # update current month or given period
if os.path.exists(file.retrieveFile()):
print 'removing %s' % file.retrieveFile()
os.remove(file.retrieveFile())
node.removeFile(file) # remove old file and create new
f = None
break
else: # old month, do nothing
print 'old file doing nothing'
return None
except:
return None
if not f:
# create new file
f_name = config.get("paths.tempdir") + "stat_{}_{}_{}.xml".format(node.id, timestamp, type)
if os.path.exists(f_name):
f = open(f_name, "a")
else:
# create new file and write header:
print 'creating writing headers %s' % f_name
f = open(f_name, "w")
f.write('<?xml version="1.0" encoding="utf-8" ?>\n')
f.write('<nodelist created="' + str(format_date(now(), "yyyy-mm-dd HH:MM:SS")) + '">\n')
if f_name not in statfiles:
statfiles.append(f_name)
return f
def runAction(self, node, op=""):
"""
The actual proccessing of the node object takes place here.
Read out the values of attrname and accesstype if any. Generate the
ACL-rule, and save it.
"""
l_date = node.get(self.get('attrname'))
if l_date:
if date.validateDateString(l_date):
try:
node.set('updatetime', date.format_date(date.parse_date(l_date)))
formated_date = date.format_date(date.parse_date(l_date), "dd.mm.yyyy")
for item in self.get('accesstype').split(';'):
node.setAccess(item, "{date >= %s}" % formated_date)
node.getLocalRead()
if self.get('recipient'): # if the recipient-email was entered, create a scheduler
attr_dict = {'single_trigger': l_date, 'function': "test_sendmail01",
'nodelist': list(node.id), 'attr_recipient': self.get('recipient'),
'attr_subject': "{} ID: {}".format(self.get('subject'),
node.id),
'attr_body': self.get('body')}
schedules.create_schedule("WorkflowStep_Defer", attr_dict)
except ValueError as e:
print "Error: %s" % e
def getStatFile(col_id, timestamp, type, period=period):
f = None
node = col_id.collection
orig_file = None
for file in node.getFiles():
if file.getType() == u"statistic":
try:
if file.getName() == u"stat_{}_{}_{}.xml".format(node.id, timestamp, type):
if timestamp == format_date(now(), "yyyy-mm") or timestamp == period: # update current month or given period
# orig_file = file.retrieveFile()
if os.path.exists(file.retrieveFile()):
print 'removing %s' % file.retrieveFile()
os.remove(file.retrieveFile())
orig_file = file.retrieveFile()
# node.files.remove(file)
f = None
break
else: # old month, do nothing
print 'old file doing nothing'
return None
except:
return None
if not f:
# create new file
f_name = config.get("paths.tempdir") + u"stat_{}_{}_{}.xml".format(node.id, timestamp, type)
# create new file and write header:j
print 'creating writing headers %s' % f_name
f = codecs.open(f_name, "w", encoding='utf8')
f.write('<?xml version="1.0" encoding="utf-8" ?>\n')
f.write('<nodelist created="' + format_date(now(), "yyyy-mm-dd HH:MM:SS") + '">\n')
if f_name not in col_id.statfiles:
col_id.statfiles.append((f_name, orig_file))
return f
def archive_thread(self):
if not time:
return
while True:
time.sleep(int(config.get("archive.interval", 60)))
archive_nodes_3 = db.getNodeIdByAttribute("archive_state", "3")
archive_nodes_2 = []
date_now = format_date(now(), "yyymmddhhmmss")
for manager in self.manager:
# search for nodes to archive after access over period (state 2)
for n in db.getNodeIdByAttribute("archive_state", "2"):
try:
node = tree.getNode(n)
if node.get("archive_date"):
date_archive = format_date(parse_date(node.get("archive_date"), "%Y-%m-%dT%H:%M:%S"), "yyymmddhhmmss")
if date_now >= date_archive:
archive_nodes_2.append(long(node.id))
except:
pass
# union to get all nodes with state 3 and 2 with over period
archive_nodes = union((archive_nodes_3, archive_nodes_2))
nodes = intersection((db.getNodeIdByAttribute("archive_type", str(manager)), archive_nodes))
# run action defined in manager
try:
self.manager[manager].actionArchive(nodes)
except:
pass
def runAction(self, node, op=""):
"""
The actual proccessing of the node object takes place here.
Read out the values of attrname and accesstype if any. Generate the
ACL-rule, and save it.
"""
l_date = node.get(self.get('attrname'))
if l_date:
if date.validateDateString(l_date):
try:
node.set('updatetime',
date.format_date(date.parse_date(l_date)))
formated_date = date.format_date(date.parse_date(l_date),
"dd.mm.yyyy")
d = formated_date.split('.')
rule = get_or_add_defer_daterange_rule(
int(d[2]), int(d[1]), int(d[0]))
for access_type in self.get('accesstype').split(';'):
special_access_ruleset = node.get_or_add_special_access_ruleset(
ruletype=access_type)
special_access_ruleset.rule_assocs.append(
AccessRulesetToRule(rule=rule))
db.session.commit()
except ValueError:
logg.exception(
"exception in workflow step defer, runAction failed")
def get_datelists(nodes):
'''
helper funtion to update default context before calling TAL interpreter
'''
update_date = []
if len(nodes) == 1:
for node in nodes:
if node.updatetime:
try:
date = parse_date(
node.updatetime, "%Y-%m-%dT%H:%M:%S")
datestr = format_date(date, format='%d.%m.%Y %H:%M:%S')
except:
datestr = node.updatetime
update_date.append([node.get("updateuser"), datestr])
creation_date = []
if len(nodes) == 1:
for node in nodes:
if node.get("creationtime"):
try:
date = parse_date(
node.get("creationtime"), "%Y-%m-%dT%H:%M:%S")
datestr = format_date(date, format='%d.%m.%Y %H:%M:%S')
except:
datestr = node.get("creationtime")
creation_date.append([node.get("creator"), datestr])
return update_date, creation_date
def findLastTwoQueuesCount():
formatter = u"%Y-%m-%d %H:" + u"%s:00" %((datetime.datetime.now().minute / 10) * 10)
date_time = date.format_date(datetime.datetime.now(), formatter)
start_date = datetime.datetime.strptime(date_time, date.FORMAT_DATE_NORMAL)
old_date = date.format_date(start_date - datetime.timedelta(seconds=600)) #十分钟之前
kwargs = {'create_time__gte':old_date}
return Monitor.objects.filter(**kwargs).order_by("-tasks_count")
def get_datelists(nodes):
'''
helper funtion to update default context before calling TAL interpreter
'''
update_date = []
if len(nodes) == 1:
for node in nodes:
if node.updatetime:
try:
date = parse_date(
node.updatetime, "%Y-%m-%dT%H:%M:%S")
datestr = format_date(date, format='%d.%m.%Y %H:%M:%S')
except:
datestr = node.updatetime
update_date.append([node.get("updateuser"), datestr])
creation_date = []
if len(nodes) == 1:
for node in nodes:
if node.get("creationtime"):
try:
date = parse_date(
node.get("creationtime"), "%Y-%m-%dT%H:%M:%S")
datestr = format_date(date, format='%d.%m.%Y %H:%M:%S')
except:
datestr = node.get("creationtime")
creation_date.append([node.get("creator"), datestr])
return update_date, creation_date
def pretty_format_update_time(self):
"""XXX: move this somewhere else!"""
if self.updatetime:
return format_date(parse_date(self.updatetime), '%d.%m.%Y, %H:%M:%S')
if self.creationtime:
return format_date(parse_date(self.creationtime), '%d.%m.%Y, %H:%M:%S')
return ''
def getUpdatedDate(self, format=None):
if format is None:
format = STANDARD_FORMAT
if self.get('updatetime'):
return format_date(parse_date(self.get('updatetime')), '%d.%m.%Y, %H:%M:%S')
if self.get('creationtime'):
return format_date(parse_date(self.get('creationtime')), '%d.%m.%Y, %H:%M:%S')
return ''
def replace_or_add_defer_date(node, iso_date_old, iso_date_new, access_types=[u'read', u'data']):
"""
examples:
# set new date in the future (for read and data access)
replace_or_add_defer_date(node, u"2016-11-03T00:00:00", u"2017-11-03T00:00:00")
# remove existing rule
replace_or_add_defer_date(node, u"2016-11-03T00:00:00", u"")
# add new rule
replace_or_add_defer_date(node, u"", u"2017-11-03T00:00:00")
"""
if (not (iso_date_old or iso_date_new)) or (not access_types):
print ("noting to be done")
return
import utils.date as date
from psycopg2.extras import DateRange
import datetime
from core import AccessRule, AccessRulesetToRule, db
q = db.query
from core.permission import get_or_add_access_rule
if iso_date_old:
formatted_date_old = date.format_date(date.parse_date(iso_date_old), "dd.mm.yyyy")
day_old, month_old, year_old = map(int, formatted_date_old.split('.'))
dateranges_old = set([DateRange(datetime.date(year_old, month_old, day_old),
datetime.date(9999, 12, 31), '[)')])
# !!! exit if more than one rule for this daterange exists, should be fixed by looping through the nodes rules
rule_old = q(AccessRule).filter_by(group_ids=None, dateranges=dateranges_old, subnets=None,
invert_group=False, invert_date=False, invert_subnet=False).one()
if not rule_old:
raise ValueError('rule not found for old date {}'.format(iso_date_old))
print("found old rule {}".format(rule_old.to_dict()))
else:
rule_old = None
if iso_date_new:
formatted_date_new = date.format_date(date.parse_date(iso_date_new), "dd.mm.yyyy")
day_new, month_new, year_new = map(int, formatted_date_new.split('.'))
dateranges_new = set([DateRange(datetime.date(year_new, month_new, day_new),
datetime.date(9999, 12, 31), '[)')])
rule_new = get_or_add_access_rule(dateranges=dateranges_new)
else:
rule_new = None
for access_type in access_types:
special_access_ruleset = node.get_or_add_special_access_ruleset(ruletype=access_type)
if rule_old:
arr_old = q(AccessRulesetToRule).filter_by(ruleset=special_access_ruleset, rule=rule_old).one()
special_access_ruleset.rule_assocs.remove(arr_old)
if rule_new:
special_access_ruleset.rule_assocs.append(AccessRulesetToRule(rule=rule_new))
def pretty_format_update_time(self):
"""XXX: move this somewhere else!"""
if self.updatetime:
return format_date(parse_date(self.updatetime),
'%d.%m.%Y, %H:%M:%S')
if self.creationtime:
return format_date(parse_date(self.creationtime),
'%d.%m.%Y, %H:%M:%S')
return ''
def replace_or_add_defer_date(node, iso_date_old, iso_date_new, access_types=[u'read', u'data']):
"""
examples:
# set new date in the future (for read and data access)
replace_or_add_defer_date(node, u"2016-11-03T00:00:00", u"2017-11-03T00:00:00")
# remove existing rule
replace_or_add_defer_date(node, u"2016-11-03T00:00:00", u"")
# add new rule
replace_or_add_defer_date(node, u"", u"2017-11-03T00:00:00")
"""
if (not (iso_date_old or iso_date_new)) or (not access_types):
print ("noting to be done")
return
import utils.date as date
from psycopg2.extras import DateRange
import datetime
from core import AccessRule, AccessRulesetToRule, db
q = db.query
from core.permission import get_or_add_access_rule
if iso_date_old:
formatted_date_old = date.format_date(date.parse_date(iso_date_old), "dd.mm.yyyy")
day_old, month_old, year_old = map(int, formatted_date_old.split('.'))
dateranges_old = set([DateRange(datetime.date(year_old, month_old, day_old),
datetime.date(9999, 12, 31), '[)')])
# !!! exit if more than one rule for this daterange exists, should be fixed by looping through the nodes rules
rule_old = q(AccessRule).filter_by(group_ids=None, dateranges=dateranges_old, subnets=None,
invert_group=False, invert_date=False, invert_subnet=False).one()
if not rule_old:
raise ValueError('rule not found for old date {}'.format(iso_date_old))
print("found old rule {}".format(rule_old.to_dict()))
else:
rule_old = None
if iso_date_new:
formatted_date_new = date.format_date(date.parse_date(iso_date_new), "dd.mm.yyyy")
day_new, month_new, year_new = map(int, formatted_date_new.split('.'))
dateranges_new = set([DateRange(datetime.date(year_new, month_new, day_new),
datetime.date(9999, 12, 31), '[)')])
rule_new = get_or_add_access_rule(dateranges=dateranges_new)
else:
rule_new = None
for access_type in access_types:
special_access_ruleset = node.get_or_add_special_access_ruleset(ruletype=access_type)
if rule_old:
arr_old = q(AccessRulesetToRule).filter_by(ruleset=special_access_ruleset, rule=rule_old).one()
special_access_ruleset.rule_assocs.remove(arr_old)
if rule_new:
special_access_ruleset.rule_assocs.append(AccessRulesetToRule(rule=rule_new))
def getUpdatedDate(self, format=None):
if format is None:
format = STANDARD_FORMAT
if self.get('updatetime'):
return format_date(parse_date(self.get('updatetime')),
'%d.%m.%Y, %H:%M:%S')
if self.get('creationtime'):
return format_date(parse_date(self.get('creationtime')),
'%d.%m.%Y, %H:%M:%S')
return ''
def try_node_date(node, attr_list=attr_list):
for key in attr_list:
timevalue = node.get(key)
if date_pattern_standard.match(timevalue):
rfc822_time = format_date(parse_date(timevalue), 'rfc822')
break
elif date_pattern_armus.match(timevalue):
rfc822_time = format_date(parse_date(timevalue, format="%m/%d/%Y"), 'rfc822')
break
else:
rfc822_time = ''
return rfc822_time
def view(req):
page = req.params.get("page", "")
gotopage = req.params.get("gotopage", "")
root = getRoot()
v = {}
v["gotopage"] = gotopage
print "overview", gotopage, req.params
def format_num(nums):
num = ""
for k in nums:
num += k + ":" + str(nums[k]) + ";"
return num
if gotopage == "overview" and req.params.get("changes") == "overview":
print "changes"
for key in req.params.keys():
if key == "overview_reset":
root.set("admin.stats.num", format_num(getOverviewData(req)))
root.set("admin.stats.updatetime", str(format_date()))
break
if page == "overview":
num = root.get("admin.stats.num")
if num == "":
root.set("admin.stats.num", format_num(getOverviewData(req)))
root.set("admin.stats.updatetime", str(format_date()))
n = {}
for items in num[:-1].split(";"):
n[items.split(":")[0]] = items.split(":")[1]
v['num'] = n
v['date'] = root.get("admin.stats.updatetime")
return req.getTAL("web/admin/modules/stats.html",
v,
macro="view_overview")
elif page == "type":
return req.getTAL("web/admin/modules/stats.html", v, macro="view_type")
elif page == "size":
return req.getTAL("web/admin/modules/stats.html", v, macro="view_size")
else:
return req.getTAL("web/admin/modules/stats.html", v, macro="view")
def getFormatedValue(self, field, node, language=None, html=1):
value = node.get(field.getName())
if not value or value == "0000-00-00T00:00:00": # dummy for unknown
return (field.getLabel(), "")
else:
try:
d = parse_date(value)
except ValueError:
return (field.getLabel(), value)
value = format_date(d, format=field.getValues())
value_list = []
if re.search(r'\d{2}\W\d{2}\W', value):
day_month = re.sub(r'00\W', '', re.search(r'\d{2}\W\d{2}\W', value).group())
value_list.append(day_month)
if re.search(r'\d{4}\W\d{2}', value):
year_month = re.sub(r'\W00', '', re.search(r'\d{4}-\d{2}', value).group())
value_list.append(year_month)
elif re.search(r'\d{4}', value):
value_list.append(re.search(r'\d{4}', value).group())
return (field.getLabel(), ''.join(value_list))
def formatdate(value, f='%d.%m.%Y %H:%M:%S'):
if not isinstance(value, unicode):
value = unicode(value)
try:
return format_date(parse_date(value, "%Y-%m-%dT%H:%M:%S"), format=f)
except ValueError:
return value
def getFormattedValue(self, metafield, maskitem, mask, node, language, html=True):
''' search with re if string could be a date
appends this to a list and returns this
:param metafield: metadatafield
:param node: node with fields
:return: formatted value
'''
value = node.get(metafield.getName())
if not value or value == "0000-00-00T00:00:00": # dummy for unknown
return (metafield.getLabel(), u"")
else:
try:
d = parse_date(value)
except ValueError:
return (metafield.getLabel(), value)
value = format_date(d, format=metafield.getValues())
value_list = []
if re.search(r'\d{2}\W\d{2}\W', value):
day_month = re.sub(r'00\W', '', re.search(r'\d{2}\W\d{2}\W', value).group())
value_list.append(day_month)
if re.search(r'\d{4}\W\d{2}', value):
year_month = re.sub(r'\W00', '', re.search(r'\d{4}-\d{2}', value).group())
value_list.append(year_month)
elif re.search(r'\d{4}', value):
value_list.append(re.search(r'\d{4}', value).group())
return (metafield.getLabel(), ''.join(value_list))
def formatdate(value, f='%d.%m.%Y %H:%M:%S'):
if not isinstance(value, unicode):
value = unicode(value)
try:
return format_date(parse_date(value, "%Y-%m-%dT%H:%M:%S"), format=f)
except ValueError:
return value
def show_workflow_node(self, node, req):
attrname = self.get("attrname")
niss = self.get("niss")
if attrname == "":
attrname = "urn"
# create urn only for nodes with files
if len(node.files) > 0:
urn = node.get(attrname)
if urn:
node.set(attrname, utilsurn.increaseURN(node.get(attrname)))
else:
for var in re.findall(r'\[(.+?)\]', niss):
if var == "att:id":
niss = niss.replace("[" + var + "]", unicode(node.id))
elif var.startswith("att:"):
val = node.get(var[4:])
try:
val = date.format_date(date.parse_date(val),
'%Y%m%d')
except:
logg.exception(
"exception in workflow step urn, date formatting failed, ignoring"
)
niss = niss.replace("[" + var + "]", val)
node.set(
attrname,
utilsurn.buildNBN(self.get("snid1"), self.get("snid2"),
niss))
db.session.commit()
return self.forwardAndShow(node, True, req)
def show_workflow_node(self, node, req):
attrname = self.get("attrname")
niss = self.get("niss")
if attrname == "":
attrname = "urn"
# create urn only for nodes with files
if len(node.getFiles()) > 0:
urn = node.get(attrname)
if urn:
node.set(attrname, utilsurn.increaseURN(node.get(attrname)))
else:
for var in re.findall(r"\[(.+?)\]", niss):
if var == "att:id":
niss = niss.replace("[" + var + "]", node.id)
elif var.startswith("att:"):
val = node.get(var[4:])
try:
val = date.format_date(date.parse_date(val), "%Y%m%d")
except:
pass
niss = niss.replace("[" + var + "]", val)
node.set(attrname, utilsurn.buildNBN(self.get("snid1"), self.get("snid2"), niss))
return self.forwardAndShow(node, True, req)
def show_workflow_step(self, req):
if not self.has_write_access():
return '<i>' + t(lang(req), "permission_denied") + '</i>'
c = []
display_name_attr = self.parents[0].display_name_attribute
i = 0
for item in self.children:
c.append({
"id":
unicode(item.id),
"creationtime":
date.format_date(date.parse_date(item.get('creationtime')),
'dd.mm.yyyy HH:MM:SS')
})
if display_name_attr:
c[i]["name"] = item.get(display_name_attr)
else:
c[i]["name"] = item.name
i += 1
c.sort(lambda x, y: cmp(x['name'], y['name']))
return req.getTAL("workflow/workflow.html", {
"children": c,
"workflow": self.parents[0],
"step": self,
"nodelink": "/mask?id={}&obj=".format(self.id),
'currentlang': lang(req),
"csrf": req.csrf_token.current_token
},
macro="workflow_show")
def show_workflow_node(self, node, req):
attrname = self.get("attrname")
niss = self.get("niss")
if attrname == "":
attrname = "urn"
# create urn only for nodes with files
if len(node.files) > 0:
urn = node.get(attrname)
if urn:
node.set(attrname, utilsurn.increaseURN(node.get(attrname)))
else:
for var in re.findall(r'\[(.+?)\]', niss):
if var == "att:id":
niss = niss.replace("[" + var + "]", unicode(node.id))
elif var.startswith("att:"):
val = node.get(var[4:])
try:
val = date.format_date(date.parse_date(val), '%Y%m%d')
except:
logg.exception("exception in workflow step urn, date formatting failed, ignoring")
niss = niss.replace("[" + var + "]", val)
node.set(attrname, utilsurn.buildNBN(self.get("snid1"), self.get("snid2"), niss))
db.session.commit()
return self.forwardAndShow(node, True, req)
def show_workflow_step(self, req):
access = acl.AccessData(req)
if not access.hasWriteAccess(self):
return "<i>" + t(lang(req), "permission_denied") + "</i>"
c = []
for item in self.getChildren():
c.append(
{
"id": str(item.id),
"creationtime": date.format_date(date.parse_date(item.get("creationtime")), "dd.mm.yyyy HH:MM:SS"),
"name": item.getName(),
}
)
c.sort(lambda x, y: cmp(x["name"], y["name"]))
return req.getTAL(
"workflow/workflow.html",
{
"children": c,
"workflow": self.getParents()[0],
"step": self,
"nodelink": "/mask?id=" + self.id + "&obj=",
"currentlang": lang(req),
},
macro="workflow_show",
)
def createNewVersion(self, user):
if self.get('system.version.id') == '':
self.set('system.version.id', '1')
n = Node(name=self.name, type=self.type)
n.set("creator", self.get('creator'))
n.set("creationtime", self.get('creationtime'))
n.set("updateuser", user.getName())
n.set("edit.lastmask", self.get('edit.lastmask'))
if self.get('updatetime') < str(now()):
n.set("updatetime", str(format_date()))
else:
n.set("updatetime", str(self.get('updatetime')))
for f in self.getFiles():
n.addFile(f)
activeNode = self.getActiveVersion()
for pid in db.getParents(activeNode.id):
parentNode = getNode(pid)
parentNode.addChild(n)
parentNode.removeChild(activeNode)
for cid in db.getChildren(activeNode.id):
if cid != activeNode.prev_nid:
n.addChild(getNode(cid))
n.set("system.version.id", self.getLastVersionID() + 1)
n.setPrevID(activeNode.id)
activeNode.setNextID(n.id)
n.addChild(activeNode)
return n
def createLDAPUser(data, uname):
user = tree.Node(uname, "user")
user.set("lastname", self._getAttribute(config.get("ldap.user_lastname"), data))
user.set("firstname", self._getAttribute(config.get("ldap.user_firstname"), data))
user.set("email", self._getAttribute(config.get("ldap.user_email"), data))
user.set("organisation", self._getAttribute(config.get("ldap.user_org"), data))
user.set("comment", self._getAttribute(config.get("ldap.user_comment"), data))
user.set("identificator", self._getAttribute(config.get("ldap.user_identificator"), data))
user.set("ldapuser.creationtime", date.format_date())
if user.get("lastname") != "" and user.get("firstname") != "":
user.setName("%s %s" % (user.get("lastname"), user.get("firstname")))
added_to_groups = 0
for group in self._getAttribute(config.get("ldap.user_group"), data, ",").split(","):
if group != "" and not usergroups.existGroup(group):
#res = usergroups.create_group(group, description="LDAP Usergroup", option="")
#res.set("ldapusergroup.creationtime", date.format_date())
logging.getLogger('usertracing').info("skipped creation of ldap user group: " + group)
continue
g = usergroups.getGroup(group)
if g:
g.addChild(user)
added_to_groups += 1
logging.getLogger('usertracing').info("created ldap user: "******"created ldap user %r, %r not added to any group" % (uname, user.id))
return user
def createNewVersion(self, user):
if self.get('system.version.id') == '':
self.set('system.version.id', '1')
n = Node(name=self.name, type=self.type)
n.set("creator", self.get('creator'))
n.set("creationtime", self.get('creationtime'))
n.set("updateuser", user.getName())
n.set("edit.lastmask", self.get('edit.lastmask'))
if self.get('updatetime') < str(now()):
n.set("updatetime", str(format_date()))
else:
n.set("updatetime", str(self.get('updatetime')))
for f in self.getFiles():
n.addFile(f)
activeNode = self.getActiveVersion()
for pid in db.getParents(activeNode.id):
parentNode = getNode(pid)
parentNode.addChild(n)
parentNode.removeChild(activeNode)
for cid in db.getChildren(activeNode.id):
if cid != activeNode.prev_nid:
n.addChild(getNode(cid))
n.set("system.version.id", self.getLastVersionID() + 1)
n.setPrevID(activeNode.id)
activeNode.setNextID(n.id)
n.addChild(activeNode)
return n
def getDuration(self):
duration = self.get("duration")
try:
duration = float(duration)
except ValueError:
return 0
else:
return format_date(make_date(0, 0, 0, int(duration) / 3600, duration / 60, int(duration % 60)), '%H:%M:%S')
def view(req):
page = req.params.get("page", "")
gotopage = req.params.get("gotopage", "")
root = getRoot()
v = {}
v["gotopage"] = gotopage
print "overview", gotopage, req.params
def format_num(nums):
num = ""
for k in nums:
num += k + ":" + str(nums[k]) + ";"
return num
if gotopage == "overview" and req.params.get("changes") == "overview":
print "changes"
for key in req.params.keys():
if key == "overview_reset":
root.set("admin.stats.num", format_num(getOverviewData(req)))
root.set("admin.stats.updatetime", str(format_date()))
break
if page == "overview":
num = root.get("admin.stats.num")
if num == "":
root.set("admin.stats.num", format_num(getOverviewData(req)))
root.set("admin.stats.updatetime", str(format_date()))
n = {}
for items in num[:-1].split(";"):
n[items.split(":")[0]] = items.split(":")[1]
v['num'] = n
v['date'] = root.get("admin.stats.updatetime")
return req.getTAL("web/admin/modules/stats.html", v, macro="view_overview")
elif page == "type":
return req.getTAL("web/admin/modules/stats.html", v, macro="view_type")
elif page == "size":
return req.getTAL("web/admin/modules/stats.html", v, macro="view_size")
else:
return req.getTAL("web/admin/modules/stats.html", v, macro="view")
def getEditorHTML(self, field, value="", width=400, lock=0, language=None, required=None):
d = field.getSystemFormat(field.getValues())
if value == "?":
value = date.format_date(date.now(), d.getValue())
with suppress(Exception, warn=False):
value = date.format_date(date.parse_date(value), d.getValue())
return tal.getTAL("metadata/date.html", {"lock": lock,
"value": value,
"width": width,
"name": field.getName(),
"field": field,
"pattern": self.get_input_pattern(field),
"title": self.get_input_title(field),
"placeholder": self.get_input_placeholder(field),
"required": self.is_required(required)},
macro="editorfield",
language=language)
def update_node(req, path, params, data, id):
# get the user and verify the signature
if params.get('user'):
user = users.getUser(params.get('user'))
userAccess = AccessData(user=user)
if userAccess.user:
valid = userAccess.verify_request_signature(req.fullpath, params)
if not valid:
userAccess = None
else:
userAccess = None
else:
user = users.getUser('Gast')
userAccess = AccessData(user=user)
node = tree.getNode(id)
# check user access
if userAccess and userAccess.hasAccess(node, "write"):
pass
else:
s = "No Access"
req.write(s)
d = {
'status': 'fail',
'html_response_code': '403',
'errormessage': 'no access'
}
return d['html_response_code'], len(s), d
node.name = params.get('name')
metadata = json.loads(params.get('metadata'))
# set provided metadata
for key, value in metadata.iteritems():
node.set(u(key), u(value))
# service flags
node.set("updateuser", user.getName())
node.set("updatetime", format_date())
node.setDirty()
d = {
'status': 'OK',
'html_response_code': '200',
'build_response_end': time.time()
}
s = "OK"
# we need to write in case of POST request, send as buffer wil not work
req.write(s)
req.reply_headers['updatetime'] = node.get('updatetime')
return d['html_response_code'], len(s), d
def get_iptc_tags(image_path, tags=None):
"""
get the IPTC tags/values from a given
image file
:rtype : object
:param image_path: path to the image file
:param tags: dictionary with wanted iptc tags
:return: dictionary with tag/value
"""
if tags == None:
tags = get_wanted_iptc_tags()
if not isinstance(tags, dict):
logg.warn('No Tags to read.')
return
if image_path is None:
logg.warn('No file path for reading iptc.')
return
if not os.path.exists(image_path):
logg.warn('Could not read IPTC metadata from non existing file.')
return
if os.path.basename(image_path).startswith('-'):
logg.warn('Will not read IPTC metadata to files starting with a hyphen, caused by exiftool security issues.')
return
# fetch metadata dict from exiftool
exiftool_exe = config.get("external.exiftool", "exiftool")
with exiftool.ExifTool(exiftool_exe) as et:
iptc_metadata = et.get_metadata(image_path)
ret = {}
for iptc_tag in tags.keys():
key = "IPTC:" + iptc_tag
if key in iptc_metadata:
value = iptc_metadata[key]
# format dates for date fields
if iptc_tag == 'DateCreated':
if validateDate(parse_date(value, format='%Y:%m:%d')):
value = format_date(parse_date(value, format='%Y:%m:%d'))
else:
logg.error('Could not validate: {} as date value.'.format(value))
# join lists to strings
if isinstance (value, list):
value = ';'.join(ensure_unicode(e, silent=True) for e in value)
ret[iptc_tag] = ensure_unicode(value, silent=True)
logg.info('{} read from file.'.format(ret))
return ret
def write_iptc_tags(image_path, tag_dict):
'''
Writes iptc tags with exiftool to a
given image path (overwrites the sourcefile).
Emty tags (tagname='') will be removed.
:param image_path: imaqe path to write
:param tag_dict: tagname / tagvalue
:return status
'''
try:
utils.process.call(['exiftool'])
except OSError:
logg.error('No exiftool installed.')
return
image_path = os.path.abspath(image_path)
if not os.path.exists(image_path):
logg.info(u'Image {} for writing IPTC metadata does not exist.'.format(image_path))
return
if not isinstance(tag_dict, dict):
logg.error(u'No dictionary of tags.')
return
command_list = [u'exiftool']
command_list.append(u'-overwrite_original')
command_list.append(u'-charset')
command_list.append(u'iptc=UTF8')
command_list.append(image_path)
for tag_name in tag_dict.keys():
tag_value = tag_dict[tag_name]
if tag_dict[tag_name] == '':
command_list.append(u'-{}='.format(tag_name))
elif tag_name == u'DateCreated':
if validateDate(parse_date(tag_value.split('T')[0], format='%Y-%m-%d')):
tag_value = format_date(parse_date(tag_value.split('T')[0], format='%Y-%m-%d'), '%Y:%m:%d')
else:
logg.error(u'Could not validate {}.'.format(tag_value))
command_list.append(u'-charset iptc=UTF8')
command_list.append(u'-{}={}'.format(tag_name, tag_value))
logg.info(u'Command: {} will be executed.'.format(command_list))
process = utils.process.Popen(command_list, stdout=subprocess.PIPE)
output, error = process.communicate()
if error is not None:
logg.info('Exiftool output: {}'.format(output))
logg.error('Exiftool error: {}'.format(error))
def indexer_thread(timewait):
if timewait < 10:
timewait = 10
while True:
time.sleep(timewait - 10)
log.info("Re-indexing Database")
time.sleep(10)
makeSearchIndex()
log.info("Re-indexing Database: done")
tree.getRoot().set("lastindexerrun", date.format_date())
def _prepare_response():
res = {}
res['build_response_start'] = time.time()
retrievaldate = format_date()
res['retrievaldate'] = retrievaldate
res['method'] = request.command
res['path'] = request.path
res['query'] = request.query
res['timetable'] = []
return res
def indexer_thread(timewait):
if timewait < 10:
timewait = 10
while True:
time.sleep(timewait - 10)
log.info("Re-indexing Database")
time.sleep(10)
makeSearchIndex()
log.info("Re-indexing Database: done")
tree.getRoot().set("lastindexerrun", date.format_date())
def update_node(req, path, params, data, id):
# get the user and verify the signature
if params.get('user'):
user = users.getUser(params.get('user'))
userAccess = AccessData(user=user)
if userAccess.user:
valid = userAccess.verify_request_signature(req.fullpath, params)
if not valid:
userAccess = None
else:
userAccess = None
else:
user = users.getUser('Gast')
userAccess = AccessData(user=user)
node = tree.getNode(id)
# check user access
if userAccess and userAccess.hasAccess(node, "write"):
pass
else:
s = "No Access"
req.write(s)
d = {
'status': 'fail',
'html_response_code': '403',
'errormessage': 'no access'}
return d['html_response_code'], len(s), d
node.name = params.get('name')
metadata = json.loads(params.get('metadata'))
# set provided metadata
for key, value in metadata.iteritems():
node.set(u(key), u(value))
# service flags
node.set("updateuser", user.getName())
node.set("updatetime", format_date())
node.setDirty()
d = {
'status': 'OK',
'html_response_code': '200',
'build_response_end': time.time()}
s = "OK"
# we need to write in case of POST request, send as buffer wil not work
req.write(s)
req.reply_headers['updatetime'] = node.get('updatetime')
return d['html_response_code'], len(s), d
def runAction(self, node, op=""):
"""
The actual proccessing of the node object takes place here.
Read out the values of attrname and accesstype if any. Generate the
ACL-rule, and save it.
"""
l_date = node.get(self.get('attrname'))
if l_date:
if date.validateDateString(l_date):
try:
node.set('updatetime',
date.format_date(date.parse_date(l_date)))
formated_date = date.format_date(date.parse_date(l_date),
"dd.mm.yyyy")
for item in self.get('accesstype').split(';'):
node.setAccess(item, "{date >= %s}" % formated_date)
node.getLocalRead()
if self.get(
'recipient'
): # if the recipient-email was entered, create a scheduler
attr_dict = {
'single_trigger':
l_date,
'function':
"test_sendmail01",
'nodelist':
list(node.id),
'attr_recipient':
self.get('recipient'),
'attr_subject':
"{} ID: {}".format(self.get('subject'), node.id),
'attr_body':
self.get('body')
}
schedules.create_schedule("WorkflowStep_Defer",
attr_dict)
except ValueError as e:
print "Error: %s" % e
def get_cert_from_file(cert_file):
""" Return certificate information from cert_file. """
raw_subject = openssl(["x509", "-subject", "-noout", "-in %s" % cert_file])
raw_date = openssl(["x509", "-enddate", "-noout", "-in %s" % cert_file])
date = format_date(re.search('^.*=(.*)', raw_date).group(1))
domain_name = re.search('CN=(.*)/', raw_subject).group(1)
return {"domain_name": domain_name, "date": date}
def getDuration(self):
duration = self.get("duration")
try:
duration = float(duration)
except ValueError:
return 0
else:
return format_date(
make_date(0, 0, 0,
int(duration) / 3600, duration / 60,
int(duration % 60)), '%H:%M:%S')
def show_workflow_step(self, req):
access = acl.AccessData(req)
if not access.hasWriteAccess(self):
return '<i>' + t(lang(req), "permission_denied") + '</i>'
c = []
for item in self.getChildren():
c.append({"id": str(item.id), "creationtime": date.format_date(
date.parse_date(item.get('creationtime')), 'dd.mm.yyyy HH:MM:SS'), "name": item.getName()})
c.sort(lambda x, y: cmp(x['name'], y['name']))
return req.getTAL("workflow/workflow.html", {"children": c, "workflow": self.getParents()[
0], "step": self, "nodelink": "/mask?id=" + self.id + "&obj=", 'currentlang': lang(req)}, macro="workflow_show")
def creationtime(self):
# Migrated versions may contain a creationtime attribute that must be used instead because
# transaction.issued_at is set to the migration time, not creation time.
legacy_updatetime = self.attrs.get("creationtime")
if legacy_updatetime:
return legacy_updatetime
continuum_time = self.versions[0].transaction.issued_at
if continuum_time is not None:
return format_date(continuum_time)
return u""
def getEditorHTML(self, field, value="", width=400, lock=0, language=None, required=None):
d = field.getSystemFormat(str(field.getValues()))
if value == "?":
value = date.format_date(date.now(), d.getValue())
try:
value = date.format_date(date.parse_date(value), d.getValue())
except:
pass
return tal.getTAL("metadata/date.html", {"lock": lock,
"value": value,
"width": width,
"name": field.getName(),
"field": field,
"pattern": self.get_input_pattern(field),
"title": self.get_input_title(field),
"placeholder": self.get_input_placeholder(field),
"required": self.is_required(required)},
macro="editorfield",
language=language)
def format_request_value_for_db(self, field, params, item, language=None):
value = params.get(item)
f = field.getSystemFormat(str(field.getValues()))
if not f:
return ""
try:
d = parse_date(str(value), f.getValue())
except ValueError:
return ""
if not validateDate(d):
return ""
return format_date(d, format='%Y-%m-%dT%H:%M:%S')
def getFormatedValue(self, field, node, language=None, html=1):
value = node.get(field.getName())
if not value or value == "0000-00-00T00:00:00": # dummy for unknown
return (field.getLabel(), "")
else:
try:
d = parse_date(value)
except ValueError:
return (field.getLabel(), value)
value = format_date(d, format=field.getValues())
return (field.getLabel(), value)
def format_request_value_for_db(self, field, params, item, language=None):
value = params.get(item)
f = field.getSystemFormat(ustr(field.getValues()))
if not f:
return ""
try:
d = parse_date(ustr(value), f.getValue())
except ValueError:
return ""
if not validateDate(d):
return ""
return format_date(d, format='%Y-%m-%dT%H:%M:%S')
def creationtime(self):
# Migrated versions may contain a creationtime attribute that must be used instead because
# transaction.issued_at is set to the migration time, not creation time.
legacy_updatetime = self.attrs.get("creationtime")
if legacy_updatetime:
return legacy_updatetime
continuum_time = self.versions[0].transaction.issued_at
if continuum_time is not None:
return format_date(continuum_time)
return u""
def getStatFile(node, timestamp, type, period=period):
f = None
for file in node.getFiles():
if file.getType() == "statistic":
try:
if file.getName() == "stat_{}_{}_{}.xml".format(
node.id, timestamp, type):
if timestamp == str(
format_date(now(), "yyyy-mm")
) or timestamp == period: # update current month or given period
if os.path.exists(file.retrieveFile()):
print 'removing %s' % file.retrieveFile()
os.remove(file.retrieveFile())
node.removeFile(
file) # remove old file and create new
f = None
break
else: # old month, do nothing
print 'old file doing nothing'
return None
except:
return None
if not f:
# create new file
f_name = config.get("paths.tempdir") + "stat_{}_{}_{}.xml".format(
node.id, timestamp, type)
if os.path.exists(f_name):
f = open(f_name, "a")
else:
# create new file and write header:
print 'creating writing headers %s' % f_name
f = open(f_name, "w")
f.write('<?xml version="1.0" encoding="utf-8" ?>\n')
f.write('<nodelist created="' +
str(format_date(now(), "yyyy-mm-dd HH:MM:SS")) +
'">\n')
if f_name not in statfiles:
statfiles.append(f_name)
return f
def getStatFile(col_id, timestamp, type, period=period):
f = None
node = col_id.collection
orig_file = None
for file in node.getFiles():
if file.getType() == u"statistic":
try:
if file.getName() == u"stat_{}_{}_{}.xml".format(
node.id, timestamp, type):
if timestamp == format_date(
now(), "yyyy-mm"
) or timestamp == period: # update current month or given period
# orig_file = file.retrieveFile()
if os.path.exists(file.retrieveFile()):
print 'removing %s' % file.retrieveFile()
os.remove(file.retrieveFile())
orig_file = file.retrieveFile()
# node.files.remove(file)
f = None
break
else: # old month, do nothing
print 'old file doing nothing'
return None
except:
return None
if not f:
# create new file
f_name = config.get("paths.tempdir") + u"stat_{}_{}_{}.xml".format(
node.id, timestamp, type)
# create new file and write header:j
print 'creating writing headers %s' % f_name
f = codecs.open(f_name, "w", encoding='utf8')
f.write('<?xml version="1.0" encoding="utf-8" ?>\n')
f.write('<nodelist created="' +
format_date(now(), "yyyy-mm-dd HH:MM:SS") + '">\n')
if f_name not in col_id.statfiles:
col_id.statfiles.append((f_name, orig_file))
return f
def getContent(req, ids):
if len(ids) > 0:
ids = ids[0]
user = users.getUserFromRequest(req)
node = tree.getNode(ids)
access = acl.AccessData(req)
if "statsfiles" in users.getHideMenusForUser(
user) or not access.hasWriteAccess(node):
req.setStatus(httpstatus.HTTP_FORBIDDEN)
return req.getTAL("web/edit/edit.html", {}, macro="access_error")
if "update_stat" in req.params.keys(): # reset stored statistics data
msg = "user %r requests update of of system.statscontent for node %r (%r, %r)" % (
user.getName(), node.id, node.name, node.type)
logger.info(msg)
logging.getLogger('editor').info(msg)
node.removeAttribute("system.statscontent")
node.removeAttribute("system.statsdate")
# content
if req.params.get("style", "") == "popup":
statstring = node.get("system.statscontent")
if statstring == "": # load stats from objects/renew stat
data = StatTypes()
for n in node.getAllChildren():
found_dig = 0 or len([
file for file in n.getFiles()
if file.type in ["image", "document", "video"]
])
data.addItem(n.getContentType(), n.getSchema(), found_dig)
node.set("system.statscontent", str(data))
node.set("system.statsdate", str(format_date()))
statstring = str(data)
v = {}
v["data"] = StatTypes(statstring)
v["stand"] = node.get("system.statsdate")
req.writeTAL("web/edit/modules/statsfiles.html",
v,
macro="edit_stats_popup")
return ""
return req.getTAL("web/edit/modules/statsfiles.html", {"id": ids},
macro="edit_stats")
def createUrn(node, namespace, urn_type):
"""
@param node for which the URN should be created
@param namespace of the urn; list of the namespaces can be found here: http://www.iana.org/assignments/urn-namespaces/urn-namespaces.xml
@param urn_type e.q. diss, epub, etc
"""
if node.get('urn') and (node.get('urn').strip() != ''): # keep the existing urn, if there is one
logging.getLogger('everything').info('urn already exists for node %s' % node.id)
else:
try:
d = date.parse_date(node.get('date-accepted'))
except:
d = date.now()
niss = '%s-%s-%s-0' % (urn_type, date.format_date(d, '%Y%m%d'), node.id)
node.set('urn', urn.buildNBN(namespace, config.get('urn.institutionid'), niss))
def runAction(self, node, op=""):
"""
The actual proccessing of the node object takes place here.
Read out the values of attrname and accesstype if any. Generate the
ACL-rule, and save it.
"""
l_date = node.get(self.get('attrname'))
if l_date:
if date.validateDateString(l_date):
try:
node.set('updatetime', date.format_date(date.parse_date(l_date)))
formated_date = date.format_date(date.parse_date(l_date), "dd.mm.yyyy")
d = formated_date.split('.')
rule = get_or_add_defer_daterange_rule(int(d[2]), int(d[1]), int(d[0]))
for access_type in self.get('accesstype').split(';'):
special_access_ruleset = node.get_or_add_special_access_ruleset(ruletype=access_type)
special_access_ruleset.rule_assocs.append(AccessRulesetToRule(rule=rule))
db.session.commit()
except ValueError:
logg.exception("exception in workflow step defer, runAction failed")
def getContent(req, ids):
if len(ids) > 0:
ids = ids[0]
user = current_user
node = q(Node).get(ids)
if "statsfiles" in user.hidden_edit_functions or not node.has_write_access(
):
req.setStatus(httpstatus.HTTP_FORBIDDEN)
return req.getTAL("web/edit/edit.html", {}, macro="access_error")
if "update_stat" in req.params.keys(): # reset stored statistics data
logg.info(
"user %s requests update of of system.statscontent for node %s (%s, %s)",
user.login_name, node.id, node.name, node.type)
node.removeAttribute("system.statscontent")
node.removeAttribute("system.statsdate")
# content
if req.params.get("style", "") == "popup":
statstring = node.get("system.statscontent")
if statstring == "": # load stats from objects/renew stat
data = StatTypes()
for n in node.all_children:
found_dig = 0 or len([
file for file in n.files
if file.filetype in ["image", "document", "video"]
])
data.addItem(n.type, n.schema, found_dig)
node.set("system.statscontent", unicode(data))
node.set("system.statsdate", unicode(format_date()))
statstring = unicode(data)
v = {}
v["data"] = StatTypes(statstring)
v["stand"] = node.get("system.statsdate")
req.writeTAL("web/edit/modules/statsfiles.html",
v,
macro="edit_stats_popup")
return ""
return req.getTAL("web/edit/modules/statsfiles.html", {"id": ids},
macro="edit_stats")
def makeMetaData(self, audiofile):
self.attrs["mp3.version"] = audiofile.info.version
self.attrs["mp3.layer"] = audiofile.info.layer
self.attrs["mp3.bitrate"] = u"{} kBit/s".format(audiofile.info.bitrate / 1000)
self.attrs["mp3.sample_rate"] = u"{} kHz".format(audiofile.info.sample_rate / 1000)
_s = int(audiofile.info.length % 60)
_m = audiofile.info.length / 60
_h = int(audiofile.info.length) / 3600
self.attrs["mp3.length"] = format_date(make_date(0, 0, 0, _h, _m, _s), '%Y-%m-%dT%H:%M:%S')
if audiofile.tags:
for key in audio_frames.keys():
if key in audiofile.tags.keys():
self.attrs["mp3." + audio_frames[key]] = unicode(audiofile.tags[key])
def makeMetaData(self, audiofile):
self.attrs["mp3.version"] = audiofile.info.version
self.attrs["mp3.layer"] = audiofile.info.layer
self.attrs["mp3.bitrate"] = u"{} kBit/s".format(audiofile.info.bitrate / 1000)
self.attrs["mp3.sample_rate"] = u"{} kHz".format(audiofile.info.sample_rate / 1000)
_s = int(audiofile.info.length % 60)
_m = audiofile.info.length / 60
_h = int(audiofile.info.length) / 3600
self.attrs["mp3.length"] = format_date(make_date(0, 0, 0, _h, _m, _s), '%Y-%m-%dT%H:%M:%S')
if audiofile.tags:
for key in audio_frames.keys():
if key in audiofile.tags.keys():
self.attrs["mp3." + audio_frames[key]] = unicode(audiofile.tags[key])
def updateNode(self, node):
self.removeNode(node)
err = {}
err['simple'] = []
err['ext'] = []
err['text'] = []
err['commit'] = []
if not self.nodeToSimpleSearch(node):
err['simple'].append(node.id)
if not self.nodeToExtSearch(node):
err['ext'].append(node.id)
if not self.nodeToFulltextSearch(node):
err['text'].append(node.id)
node.set("updatesearchindex", str(format_date()))
return err
