def get_mediatum_iplist(name):
"""
Fetch the IPNetworkList with given name
and return its subnets list of ip ranges.
"""
query = db.query(IPNetworkList).filter(IPNetworkList.name == name)
return query.one().subnets
def register():
email = request.form['email']
email2 = request.form['email2']
if email != email2:
flash('Emails do not match')
return redirect(request.path)
password = request.form['password']
password2 = request.form['password2']
if password != password2:
flash('Passwords do not match')
return redirect(request.path)
users = db.query(User.id).filter(User.email == email).all()
if len(users) > 0:
flash('Email already registered')
return redirect(request.path)
user = User(email, password)
db.add(user)
db.commit()
g.session['user'] = user
return redirect('/')
def configure(self):
"""
Sets up the data from the database so it can be used for setting...settings.
The DB structure sucks, so some extra processing to get the correct datatypes.
TODO: Clean up, I'm writing this at 2AM..
"""
log.info("Loading configuration from the database...")
settings = dict(db.query("""SELECT `key`, `value` FROM settings"""))
log.info("Config loaded")
log.info("HoN Version: %s Chat Port: %s Protocol: %s" % (settings['honver'], settings['chatport'], settings['chatver']))
if 'username' in settings:
acc_config['username'] = settings['username']
if 'password' in settings:
acc_config['password'] = settings['password']
if 'invis' in settings:
settings['invis'] = True if settings['invis'] == "True" else False
if 'chatport' in settings:
settings['chatport'] = int(settings['chatport'])
if 'chatver' in settings:
settings['chatver'] = int(settings['chatver'])
for key in settings:
if key in basic_config:
basic_config[key] = settings[key]
self._configure(chatport=settings['chatport'], protocol=settings['chatver'], invis=settings['invis'],
masterserver=settings['masterserver'], basicserver=settings['basicserver'], honver=settings['honver'])
def _subquery_subtree_distinct(node):
from core import db
return (db.query(t_noderelation.c.cid)
.filter(t_noderelation.c.nid == node.id)
.distinct()
.subquery())
def load(cls, did):
"""Return a device object for the given device id, or None if not found."""
try:
device = db.query(Device).filter(Device.id == did).one()
return device
except NoResultFound as e:
return None
def load(cls, uid):
"""Return a user object for the given user id, or None if not found."""
try:
user = db.query(User).filter(User.id == uid).one()
return user
except (MultipleResultsFound, NoResultFound) as e:
return None
def stop_instance(app_id, instance_id):
app = db.query(App).get(app_id)
instance = app.instance(instance_id)
if instance and instance.status == 'Running' and not instance.is_live:
docker.stop(instance.container_id)
return redirect_to_app_page(app)
def authenticate(cls, api_key):
"""Return a user and device object for the given api_key, if valid."""
try:
device = db.query(Device).filter(Device.api_key == api_key).one()
user = User.load(device.uid)
return user, device
except NoResultFound as e:
return None, None
def create_instance(app_id):
app = db.query(App).get(app_id)
image = app.image(int(request.form.get('image_id', -1)))
if image:
tasks.run_image(image.id)
return redirect_to_app_page(app)
def start_instance(app_id, instance_id):
app = db.query(App).get(app_id)
instance = app.instance(instance_id)
if instance and instance.status != 'Running':
docker.start(instance.container_id)
return redirect_to_app_page(app)
def _subquery_subtree_container(node):
from contenttypes.container import Container
from core import db
query = (db.query(
t_noderelation.c.cid).filter(t_noderelation.c.nid == node.id).join(
Container, Container.id == t_noderelation.c.cid).subquery())
return query
def _cte_subtree(node):
from core import db
query = db.query(t_noderelation.c.cid).\
filter(t_noderelation.c.nid == node.id).\
distinct().\
cte(name="subtree")
return query
class UserView(BaseAdminView):
can_delete = False
form_base_class = MediatumSecureForm
column_exclude_list = ("created", "password_hash", "salt", "comment",
"private_group", "can_edit_shoppingbag",
"can_change_password")
column_filters = ("authenticator_info", "display_name", "login_name",
"organisation", "active")
can_export = True
column_details_list = ("home_dir", "authenticator_info", "id",
"login_name", "display_name", "lastname",
"firstname", "telephone", "organisation", "comment",
"email", "password_hash", "salt", "last_login",
"active", "can_edit_shoppingbag",
"can_change_password", "created_at", "group_names")
"""
"""
column_labels = dict(group_names='Groups')
column_formatters = {
"home_dir":
lambda v, c, m, p: _link_format_node_id_column(m.home_dir.id)
if m.home_dir else None
}
column_searchable_list = ("display_name", "login_name", "organisation")
column_editable_list = ("login_name", "email")
form_excluded_columns = ("home_dir", "created", "password_hash", "salt",
"versions", "shoppingbags", "private_group",
"group_assocs")
form_overrides = {"email": StringField}
form_extra_fields = {
"groups":
QuerySelectMultipleField(
query_factory=lambda: db.query(UserGroup).order_by(UserGroup.name),
widget=form.Select2Widget(multiple=True)),
"password":
StringField(),
}
def __init__(self, session=None, *args, **kwargs):
super(UserView, self).__init__(User,
session,
category="User",
*args,
**kwargs)
def on_model_change(self, form, user, is_created):
if form.password.data and user.authenticator_info.authenticator_key == INTERNAL_AUTHENTICATOR_KEY:
user.change_password(form.password.data)
def _cte_subtree(node):
from core import db
query = db.query(t_noderelation.c.cid).\
filter(t_noderelation.c.nid == node.id).\
distinct().\
cte(name="subtree")
return query
def show_node_big(self, req, style_name=""):
# style_name is ignored
content = u""
link = node_url(self.id, files=1)
sidebar = u""
pages = self.getStartpageDict()
if self.get("system.sidebar") != "":
for sb in self.get("system.sidebar").split(";"):
if sb:
l, fn = sb.split(":")
if l == lang(req):
for f in self.getFiles():
if fn.endswith(f.getName()):
sidebar = includetemplate(
self, f.retrieveFile(), {})
if sidebar:
sidebar = req.getTAL("contenttypes/container.html",
{"content": sidebar},
macro="addcolumn")
else:
sidebar = u""
if "item" in req.params:
fname = req.params.get("item")
fname_allowed = False
# accept only filenames which starts with a number and the number is a node_id and the current_user has
# read access to this node, to avoid delivering of systemfiles like /etc/passwd
# with e.g. 604993?item=../../../../../../../../../../etc/passwd
node_id_list = self.item_file_pattern.findall(fname)
if node_id_list:
node_id = node_id_list[0]
node = db.query(Node).get(node_id)
fname_allowed = node and node.has_read_access(
user=current_user)
fpath = "{}html/{}".format(config.get("paths.datadir"), fname)
if fname_allowed and os.path.isfile(fpath):
with codecs.open(fpath, "r", encoding='utf8') as c:
content = c.read()
if sidebar:
return u'<div id="portal-column-one">{}</div>{}'.format(
content, sidebar)
return content
spn = self.getStartpageFileNode(lang(req))
if spn:
long_path = spn.retrieveFile()
if os.path.isfile(long_path) and fileIsNotEmpty(long_path):
content = includetemplate(self, long_path, {'${next}': link})
if content:
if sidebar:
return u'<div id="portal-column-one">{}</div>{}'.format(
content, sidebar)
return content
return u'{}{}'.format(content, sidebar)
def api_user_register(method, email, password):
users = db.query(User.id).filter(User.email == email).all()
if len(users) > 0:
abort(400, 'Email already registered')
user = User(email, password)
db.add(user)
db.commit()
return user
def delete_instance(app_id, instance_id):
app = db.query(App).get(app_id)
instance = app.instance(instance_id)
if instance and instance.status != 'Running' and not instance.is_live:
docker.rm(instance.container_id)
db.delete(instance)
db.commit()
return redirect_to_app_page(app)
def _subquery_subtree_container(node):
from contenttypes.container import Container
from core import db
query = (db.query(t_noderelation.c.cid)
.filter(t_noderelation.c.nid == node.id)
.join(Container, Container.id == t_noderelation.c.cid)
.subquery())
return query
def set_home_dir_permissions():
users_with_home_dir = db.query(User).filter(User.home_dir_id != None)
for user in users_with_home_dir:
private_group = user.get_or_add_private_group()
db.session.flush()
assert private_group.id
rule = AccessRule(group_ids=[private_group.id])
for ruletype in (u"read", u"write", u"data"):
special_access_ruleset = user.home_dir.get_or_add_special_access_ruleset(ruletype)
special_access_ruleset.rule_assocs.append(AccessRulesetToRule(rule=rule))
def update_mediatum_iplist(name, addresses):
"""
Create or update the IPNetworkList with give
name with the ip objects in 'addresses'.
"""
iplist = db.query(IPNetworkList).filter(IPNetworkList.name == name).scalar()
if not iplist:
iplist = IPNetworkList(name=name)
db.session.add(iplist)
iplist.subnets = addresses
db.session.commit()
def show_node_big(self, req, style_name=""):
# style_name is ignored
content = u""
link = node_url(self.id, files=1)
sidebar = u""
pages = self.getStartpageDict()
if self.get("system.sidebar") != "":
for sb in self.get("system.sidebar").split(";"):
if sb:
l, fn = sb.split(":")
if l == lang(req):
for f in self.getFiles():
if fn.endswith(f.getName()):
sidebar = includetemplate(self, f.retrieveFile(), {})
if sidebar:
sidebar = req.getTAL("contenttypes/container.html", {"content": sidebar}, macro="addcolumn")
else:
sidebar = u""
if "item" in req.params:
fname = req.params.get("item")
fname_allowed = False
# accept only filenames which starts with a number and the number is a node_id and the current_user has
# read access to this node, to avoid delivering of systemfiles like /etc/passwd
# with e.g. 604993?item=../../../../../../../../../../etc/passwd
node_id_list = self.item_file_pattern.findall(fname)
if node_id_list:
node_id = node_id_list[0]
node = db.query(Node).get(node_id)
fname_allowed = node and node.has_read_access(user=current_user)
fpath = "{}html/{}".format(config.get("paths.datadir"),
fname)
if fname_allowed and os.path.isfile(fpath):
with codecs.open(fpath, "r", encoding='utf8') as c:
content = c.read()
if sidebar:
return u'<div id="portal-column-one">{}</div>{}'.format(content,
sidebar)
return content
spn = self.getStartpageFileNode(lang(req))
if spn:
long_path = spn.retrieveFile()
if os.path.isfile(long_path) and fileIsNotEmpty(long_path):
content = includetemplate(self, long_path, {'${next}': link})
if content:
if sidebar:
return u'<div id="portal-column-one">{}</div>{}'.format(content,
sidebar)
return content
return u'{}{}'.format(content, sidebar)
def authenticate(cls, email, password):
"""Authenticate the given credentials, and return the user object if
authenticated, or None if authentication failed."""
try:
user = db.query(User).filter(User.email == email).one()
if bcrypt.hashpw(password, user.password) == user.password:
return user
else:
return None
except (MultipleResultsFound, NoResultFound) as e:
return None
def set_home_dir_permissions():
users_with_home_dir = db.query(User).filter(User.home_dir_id != None)
for user in users_with_home_dir:
private_group = user.get_or_add_private_group()
db.session.flush()
assert private_group.id
rule = AccessRule(group_ids=[private_group.id])
for ruletype in (u"read", u"write", u"data"):
special_access_ruleset = user.home_dir.get_or_add_special_access_ruleset(
ruletype)
special_access_ruleset.rule_assocs.append(
AccessRulesetToRule(rule=rule))
def get(self):
page = self.get_argument('page' , 1)
db = app.model.uc.user().find().limitPage( page , 20 )
pagination = db.getPagination()
users = db.query()
list = []
for user in users:
user['roleNames'] = app.model.uc.user.roleNamesToStr( user['id'] )
user['roleIds'] = app.model.uc.user.roleIds( user['id'] )
list.append( user )
self.render("admin/user.html" , form = self.form() , list = list , roleForm = self.roleForm() , pagination = pagination )
def check_undefined_nodeclasses(stub_undefined_nodetypes=None,
fail_if_undefined_nodetypes=None,
ignore_nodetypes=[]):
"""Checks if all nodetypes found in the database are defined as subclasses of Node.
There are 3 modes which can be selected in the config file or by the parameters:
* fail_if_undefined_nodetypes is True:
=> raise an Exception if a class if missing. Recommended.
* fail_if_undefined_nodetypes is False, stub_undefined_nodetypes is True:
=> emit a warning that classes are missing and create stub classes directly inheriting from Node.
Most code will continue to work, but it may fail if the real class overrides methods from Node.
* fail_if_undefined_nodetypes is False, stub_undefined_nodetypes is False (default):
=> just emit a warning that classes are missing
"""
from core import Node, db
known_nodetypes = set(c.__mapper__.polymorphic_identity
for c in Node.get_all_subclasses())
nodetypes_in_db = set(t[0] for t in db.query(Node.type.distinct()))
undefined_nodetypes = nodetypes_in_db - known_nodetypes - set(
ignore_nodetypes)
if undefined_nodetypes:
if fail_if_undefined_nodetypes is None:
fail_if_undefined_nodetypes = config.get(
"config.fail_if_undefined_nodetypes", "false") == "true"
msg = u"some node types are present in the database, but not defined in code. Missing plugins?\n{}".format(
undefined_nodetypes)
if fail_if_undefined_nodetypes:
raise Exception(msg)
else:
logg.warn(msg)
if stub_undefined_nodetypes is None:
stub_undefined_nodetypes = config.get(
"config.stub_undefined_nodetypes", "false") == "true"
if stub_undefined_nodetypes:
for t in undefined_nodetypes:
clsname = t.capitalize()
type(str(clsname), (Node, ), {})
logg.info("auto-generated stub class for node type '%s'",
clsname)
def go_live(app_id, instance_id):
app = db.query(App).get(app_id)
instance = app.instance(instance_id)
if instance and instance.status == 'Running':
for i in app.instances:
i.is_live = False
instance.is_live = True
f = app.app_type()
f.create_front_end(instance.container_id, app.url[7:])
db.commit()
return redirect_to_app_page(app)
def slow_content_children_for_all_subcontainers(self):
"""
!!! very slow, use content_children_for_all_subcontainers instead!!!
Collects all Content nodes in all subcontainers of this node.
This excludes content nodes that are children of other content nodes.
"""
warn("very slow, use content_children_for_all_subcontainers instead", DeprecationWarning)
from contenttypes.data import Content
from core import db
sq = _subquery_subtree_container(self)
query = db.query(Content).\
join(t_noderelation, Node.id == t_noderelation.c.cid).\
filter(t_noderelation.c.nid.in_(sq) | (t_noderelation.c.nid == self.id)).\
filter(t_noderelation.c.distance == 1)
return query
def create_image(app_id):
app = db.query(App).get(app_id)
form = create_new_image_form(app, request.form)
if form.validate():
app_image = AppImage(name=form.image_name.data, status='Pending build since %s' % datetime.now().strftime('%Y-%m-%d %H:%M:%S'), app=app)
db.add(app_image)
db.flush()
params = extract_build_params(app, app_image, form)
app_image.params_json = json.dumps(params)
db.commit()
tasks.build_image.delay(app_image.id)
return redirect_to_app_page(app)
else:
return render_template('createImage.html', app=app, current_tab='apps', form=form)
def get(self):
contentModel = app.model.bc.content()
page = self.get_argument('page',1)
db = contentModel.find().limitPage( page , 20)\
.where('[type] = %s' , 'page')\
.order('[id] DESC , [order] DESC')\
.fields('[id] , [title] , [created]')
pagination = db.getPagination()
contents = db.query()
list = []
for v in contents:
v['createdStr'] = time.strftime('%Y-%m-%d %H:%M' , time.localtime(v['created']) )
list.append( v )
self.render("admin/pages.html" , list = list , pagination = pagination )
def slow_content_children_for_all_subcontainers(self):
"""
!!! very slow, use content_children_for_all_subcontainers instead!!!
Collects all Content nodes in all subcontainers of this node.
This excludes content nodes that are children of other content nodes.
"""
warn("very slow, use content_children_for_all_subcontainers instead",
DeprecationWarning)
from contenttypes.data import Content
from core import db
sq = _subquery_subtree_container(self)
query = db.query(Content).\
join(t_noderelation, Node.id == t_noderelation.c.cid).\
filter(t_noderelation.c.nid.in_(sq) | (t_noderelation.c.nid == self.id)).\
filter(t_noderelation.c.distance == 1)
return query
def google_login_landing():
flow = get_flow()
code = request.args.get('code')
credentials = flow.step2_exchange(code)
profile = get_api(credentials).userinfo().get().execute()
email, name = profile['email'], profile['name']
user = db.query(User).get(email)
if user is None:
user = User(email=email, name=name)
db.add(user)
db.commit()
session = request.environ['beaker.session']
session['username'] = user.email
session.save()
return redirect('/')
def buildStatAll(collections,
period="",
fname=None): # period format = yyyy-mm
"""
build the statistic files with name stat_<collection_id>_yyyy-mm_<type> where type is in
'frontend', 'download' or 'edit'
:param collections: list of collections for which the statistic files should be build
if this is an empty list, all collections and their children are
fetched as an psql command
:param period: period for which the statistic files should be build, format yyyy-mm
:param fname: optional name of the logfile, default <period>.log
:return: None
"""
data = readLogFiles(period, fname)
time0 = time.time()
collection_ids = {}
for collection in collections:
print collection
in_logitem_set = False
items = [collection] + collection.all_children.all()
ids_set = Set()
for item in items:
ids_set.add(item.id)
if item.id in logitem_set:
in_logitem_set = True
if in_logitem_set:
collection_ids[collection.id] = CollectionId(ids_set, collection)
if not collections:
# read all collections and its children with a single psql command which is much more faster
# than the use of collection.all_children
import core
out = core.db.run_psql_command(
"select nid, id from node, noderelation where cid=id and" +
" nid in (select id from node where type in ('collection', 'collections'))"
+ " order by nid",
output=True,
database=config.get("database.db"))
lines = out.split('\n')
last_collection = 0
for line in lines:
if line:
collection_s, id_s = line.split('|')
collection = int(collection_s)
id = int(id_s)
if last_collection != collection:
if last_collection:
if in_logitem_set:
collection_ids[last_collection] = CollectionId(
ids_set,
db.query(Node).get(last_collection))
in_logitem_set = False
ids_set = Set()
# add also collection itself
ids_set.add(collection)
ids_set.add(id)
if id in logitem_set:
in_logitem_set = True
last_collection = collection
if last_collection:
if in_logitem_set:
collection_ids[last_collection] = CollectionId(
ids_set,
db.query(Node).get(last_collection))
time1 = time.time()
print "time to collect all %d collections: %f" % (len(collection_ids),
time1 - time0)
# in buildStatAll_ for every collection 3 filedescriptors (frontend, download, edit) may be opened
# to avoid running out of the available filedescriptors (currently 1024 per process)
# buildStatAll_ is called multiple times with a collection chunk of lower than 300 collections
collection_ids_keys = collection_ids.keys()
collection_count = len(collection_ids_keys)
collection_chunk = collection_count
n = 1
while collection_chunk > 300:
n += 1
collection_chunk = collection_count / n
collection_chunk += 1
start_idx = 0
while start_idx < collection_count:
end_idx = start_idx + collection_chunk
print "start_idx:", start_idx, "end_idx:", end_idx
buildStatAll_(collection_ids, collection_ids_keys[start_idx:end_idx],
data, period, fname)
start_idx = end_idx
def _subquery_subtree_distinct(node):
from core import db
return (db.query(t_noderelation.c.cid).filter(
t_noderelation.c.nid == node.id).distinct().subquery())
def metadatatype_access(self):
from schema.schema import Metadatatype
return db.query(Metadatatype).join(NodeToAccessRuleset).filter_by(ruleset_name=self.name).all()
def display_new_image_form(app_id):
app = db.query(App).get(app_id)
form = create_new_image_form(app)
return render_template('createImage.html', app=app, current_tab='apps', form=form)
def motd_parser(self):
""" Retrieves the dictionary of message of the day(s(?))
and replaces S2's colour formatting with html classes.
Then places any new items into the database.
"""
colour_map = ["00","1C","38","54","70","8C","A8","C4","E0","FF"]
s2colours = lambda m: '<span style="color: #' + ''.join([colour_map[int(x)] for x in m.group(1)]) + '">'
def urlfix(x):
""" Replaces urls which only contain a 'www' with a 'http://wwww'. """
if x.group(2) in ['http://', 'www']:
colour = "y"
url = x.group(1)
else:
colour = x.group(1)
url = x.group(2)
r = re.compile(r"(?<!http://)www")
if r.match(url):
url = 'http://' + ''.join(url)
return '<a href=' + url + ' class=' + colour + ' target="_blank">' + url + '</a>'
motd_data = self.motd_get()
motd_list = motd_data['motd_list']
## NOTE: This entire thing is fairly broken due to glows, and when retards use rainbows in their text.
# Iterate over the list in reverse because entries are retrieved in order newest -> oldest
# and must be entered into the database oldest -> newest.
for motd in motd_list[::-1]:
# First find any un-coloured hyperlinks and fill them with html tags.
# This regex matches any hyperlink which is not preceeded by a ^g formatter for EG.
# It is not very accurate at the moment as it will match a http which has a ^* at the end.
# http://gskinner.com/RegExr/?2u79l
r = re.compile(r"(?<=[^\^a-zA-Z])((http://|(?<!http://)www)[-a-zA-Z0-9@:%_\+.~#?&//=]+)[^\^\*]")
motd['body'] = r.sub(urlfix, motd['body'])
# Then find all hyperlinks that contain colour formatting and replace with HTML tags.
r = re.compile(r"\^([a-zA-Z])((http://|(?<!http://)www)[-a-zA-Z0-9@:%_\+.~#?&//=]+)(\^\*)")
motd['body'] = r.sub(urlfix, motd['body'])
# Find all coded colours eg ^428 and replace with inline html styling
# ''.join([color_map[int(x)] for x in r.search(msg).group(1)])
r = re.compile(r"\^([0-9]{3})")
motd['body'] = r.sub(s2colours, motd['body'])
# Replace the colours with HTML classes
# Replace ^* with </span>
motd['body'] = motd['body'].replace("^*", "</span>")
# Find all basic colour codes eg ^y or ^r or ^o and replace with inline html
r = re.compile(r"\^([a-z]{1})")
motd['body'] = r.sub(r"<span class='\1'>", motd['body'])
# Replace \r\n with <br />
motd['body'] = motd['body'].replace("\r\n", "<br />")
title_exists = db.query("""SELECT id FROM motds WHERE title = %s AND date = %s""", [motd['title'], motd['date']])
msg_exists = db.query("""SELECT id, title FROM motds WHERE body = %s AND date = %s""", [motd['body'], motd['date']])
if not title_exists:
# Check if the message was simply updated by the staff.
# If it's been changed then update it in the database automatically.
# TODO: Is this level of comparison okay?
if msg_exists:
# Title doesn't exist, but message body does, title changed.
db.execute("""UPDATE motds SET title=%s, author=%s, date=%s, body=%s WHERE id = %s""", [motd['title'], motd['author'], motd['date'], motd['body'], msg_exists[0][0]])
log.info("Updated motd #%s - %s. Title updated to %s" % (msg_exists[0][0], msg_exists[0][1], motd['title']))
elif title_exists:
log.debug("Duplicate title for motd id %s with title %s" % (title_exists[0][0], motd['title']))
# This entry is already here, possibly it could have been updated.
# Note: Seems they like to change the titles after publishing them.
if not msg_exists:
# Title exists but the msg body doesn't, so it was likely updated.
db.execute("""UPDATE motds SET title=%s, author=%s, date=%s, body=%s WHERE id = %s""", [motd['title'], motd['author'], motd['date'], motd['body'], title_exists[0][0]])
log.info("Updated motd #%s - %s. Message updated" % (title_exists[0][0], motd['title']))
if not msg_exists and not title_exists:
# Neither the title or message are there, either both are changed or this is a brand new motd
# Treat it as new for now.
# Add this motd to the database
db.execute("""INSERT INTO motds (title, author, date, body) VALUES(%s, %s, %s, %s)""", [motd['title'], motd['author'], motd['date'], motd['body']])
log.info("Added new message of the day - %s - %s" % (motd['title'], motd['date']))
# Get the image from S2 for the motd.
# Save it to static/img/motd/ if it does not exist.
image_file = motd_data['image'].split("`")[0]
image_name = re.search(r'\/([a-f0-9]+.jpg)', image_file).group(1)
if not os.path.isfile(os.path.join(config.motd_img_dir, image_name)):
urllib.urlretrieve(image_file, os.path.join(config.motd_img_dir, image_name))
# Set the image name in the database so it can be retrieved.
db.execute("""UPDATE `motd_extra` SET `value`=%s WHERE `key`=%s""", [image_name, 'image'])
log.info("New MOTD image.")
def app_details(app_id):
app = db.query(App).get(app_id)
return render_template('app.html', app=app, current_tab='apps')
class UserGroupView(BaseAdminView):
form_base_class = MediatumSecureForm
form_excluded_columns = "user_assocs"
column_details_list = [
"id", "name", "description", "hidden_edit_functions",
"is_editor_group", "is_workflow_editor_group", "is_admin_group",
"created_at", "metadatatype_access", "user_names"
]
column_searchable_list = ("name", "description")
column_filters = ("name", "description", "is_editor_group",
"is_workflow_editor_group", "is_admin_group")
can_export = True
column_labels = dict(metadatatype_access='Metadatatypes',
user_names='Users')
edit_functions = [
'acls', 'admin', 'changeschema', 'classes', 'editor', 'files',
'identifier', 'logo', 'metadata', 'search', 'searchmask', 'sortfiles',
'statsaccess', 'statsfiles', 'upload'
]
edit_function_choices = [(x, x) for x in edit_functions]
form_extra_fields = {
"users":
QuerySelectMultipleField(
query_factory=lambda: db.query(User).order_by(User.login_name),
widget=form.Select2Widget(multiple=True)),
"metadatatypes":
QuerySelectMultipleField(query_factory=lambda: db.query(Metadatatype).
order_by(Metadatatype.name),
widget=form.Select2Widget(multiple=True)),
"hidden_edit_functions":
SelectMultipleField(choices=edit_function_choices,
widget=form.Select2Widget(multiple=True)),
}
def on_form_prefill(self, form, id):
form.metadatatypes.data = form._obj.metadatatype_access
def on_model_change(self, form, usergroup, is_created):
if is_created:
""" create ruleset for group """
existing_ruleset = q(AccessRuleset).filter_by(
name=usergroup.name).scalar()
if existing_ruleset is None:
rule = get_or_add_access_rule(group_ids=[usergroup.id])
ruleset = AccessRuleset(name=usergroup.name,
description=usergroup.name)
arr = AccessRulesetToRule(rule=rule)
ruleset.rule_assocs.append(arr)
""" add/remove access to Metadatatypes """
for mt in q(Metadatatype):
nrs_list = q(NodeToAccessRuleset).filter_by(nid=mt.id).filter_by(
ruleset_name=usergroup.name).all()
if mt in form.metadatatypes.data:
if not nrs_list:
mt.access_ruleset_assocs.append(
NodeToAccessRuleset(ruleset_name=usergroup.name,
ruletype=u'read'))
else:
for nrs in nrs_list:
mt.access_ruleset_assocs.remove(nrs)
def __init__(self, session=None, *args, **kwargs):
super(UserGroupView, self).__init__(UserGroup,
session,
category="User",
*args,
**kwargs)
def buildStatAll(collections, period="", fname=None): # period format = yyyy-mm
"""
build the statistic files with name stat_<collection_id>_yyyy-mm_<type> where type is in
'frontend', 'download' or 'edit'
:param collections: list of collections for which the statistic files should be build
if this is an empty list, all collections and their children are
fetched as an psql command
:param period: period for which the statistic files should be build, format yyyy-mm
:param fname: optional name of the logfile, default <period>.log
:return: None
"""
data = readLogFiles(period, fname)
time0 = time.time()
collection_ids = {}
for collection in collections:
print collection
in_logitem_set = False
items = [collection] + collection.all_children.all()
ids_set = Set()
for item in items:
ids_set.add(item.id)
if item.id in logitem_set:
in_logitem_set = True
if in_logitem_set:
collection_ids[collection.id] = CollectionId(ids_set, collection)
if not collections:
# read all collections and its children with a single psql command which is much more faster
# than the use of collection.all_children
import core
out = core.db.run_psql_command("select nid, id from node, noderelation where cid=id and" +
" nid in (select id from node where type in ('collection', 'collections'))" +
" order by nid",
output=True, database=config.get("database.db"))
lines = out.split('\n')
last_collection = 0
for line in lines:
if line:
collection_s, id_s = line.split('|')
collection = int(collection_s)
id = int(id_s)
if last_collection != collection:
if last_collection:
if in_logitem_set:
collection_ids[last_collection] = CollectionId(ids_set, db.query(Node).get(last_collection))
in_logitem_set = False
ids_set = Set()
# add also collection itself
ids_set.add(collection)
ids_set.add(id)
if id in logitem_set:
in_logitem_set = True
last_collection = collection
if last_collection:
if in_logitem_set:
collection_ids[last_collection] = CollectionId(ids_set, db.query(Node).get(last_collection))
time1 = time.time()
print "time to collect all %d collections: %f" % (len(collection_ids), time1 - time0)
# in buildStatAll_ for every collection 3 filedescriptors (frontend, download, edit) may be opened
# to avoid running out of the available filedescriptors (currently 1024 per process)
# buildStatAll_ is called multiple times with a collection chunk of lower than 300 collections
collection_ids_keys = collection_ids.keys()
collection_count = len(collection_ids_keys)
collection_chunk = collection_count
n = 1
while collection_chunk > 300:
n += 1
collection_chunk = collection_count / n
collection_chunk += 1
start_idx = 0
while start_idx < collection_count:
end_idx = start_idx + collection_chunk
print "start_idx:", start_idx, "end_idx:", end_idx
buildStatAll_(collection_ids, collection_ids_keys[start_idx:end_idx], data, period, fname)
start_idx = end_idx
def metadatatype_access(self):
from schema.schema import Metadatatype
return db.query(Metadatatype).join(NodeToAccessRuleset).filter_by(
ruleset_name=self.name).all()
def load_by_user(cls, uid):
"""Returns a list of device objects for the given user id."""
try:
return db.query(Device).filter(Device.uid == uid).all()
except NoResultFound:
return []
