class Controller(object):
_header_name = Option("HeaderName", "rdiffweb")
_footername = Option("FooterName", "rdiffweb")
_footerurl = Option("FooterUrl", "https://www.ikus-soft.com/en/rdiffweb/")
_default_theme = Option("DefaultTheme", "default")
@property
def app(self):
return cherrypy.request.app
def _compile_template(self, template_name, **kwargs):
"""
Used to generate a standard HTML page using the given template.
This method should be used by subclasses to provide default template
value.
"""
loc = cherrypy.response.i18n.locale
parms = {
"lang": loc.language,
"header_name": self._header_name,
"theme": self._default_theme,
"footername": self._footername,
"footerurl": self._footerurl,
}
if self.app.currentuser:
parms.update({
'username': self.app.currentuser.username,
'is_admin': self.app.currentuser.is_admin,
'is_maintainer': self.app.currentuser.is_maintainer,
})
# Append custom branding
if hasattr(self.app.root, "header_logo"):
parms["header_logo"] = '/header_logo'
# Append template parameters.
parms.update(kwargs)
return self.app.templates.compile_template(template_name, **parms)
def _is_submit(self):
"""
Check if the cherrypy request is a POST.
"""
return cherrypy.request.method == "POST"
class LoginPage(Controller):
"""
This page is used by the authentication to display enter a user/pass.
"""
_welcome_msg = Option("WelcomeMsg")
def index(self, redirect=b'/', username='', error_msg='', **kwargs):
# Re-encode the redirect for display in HTML
redirect = quote_url(redirect, safe=";/?:@&=+$,%")
params = {
'redirect': redirect,
'login': username,
'warning': error_msg
}
# Add welcome message to params. Try to load translated message.
params["welcome_msg"] = self._welcome_msg
if hasattr(cherrypy.response, 'i18n'):
lang = cherrypy.response.i18n.locale.language
params["welcome_msg"] = Option(
"WelcomeMsg[%s]" % (lang),
default=params["welcome_msg"]).get()
return self._compile_template("login.html", **params).encode("utf-8")
class RemoveOlder(Deamon):
_remove_older_time = Option('RemoveOlderTime', '23:00')
def __init__(self, bus, app):
self.app = app
Deamon.__init__(self, bus)
@property
def job_execution_time(self):
return self._remove_older_time
def job_run(self):
"""
Execute the job in background.
"""
# Create a generator to loop on repositories.
gen = ((user, repo, int(repo.keepdays))
for user in self.app.store.users() for repo in user.repo_objs
if int(repo.keepdays) > 0)
# Loop on each repos.
for user, repo, keepdays in gen:
try:
self._remove_older(user, repo, keepdays)
except BaseException:
_logger.exception(
"fail to remove older for user [%r] repo [%r]", user, repo)
def _remove_older(self, user, repo, keepdays):
"""
Take action to remove older.
"""
assert isinstance(keepdays, int)
assert keepdays > 0
# Get instance of the repo.
r = librdiff.RdiffRepo(user.user_root, repo.name)
# Check history date.
if not r.last_backup_date:
_logger.info("no backup dates for [%r]", r.full_path)
return
d = librdiff.RdiffTime() - r.last_backup_date
d = d.days + keepdays
_logger.info("execute rdiff-backup --force --remove-older-than=%sD %r",
d, r.full_path)
r.execute(
b'--force',
b'--remove-older-than=' + str(d).encode(encoding='latin1') + b'D',
r.full_path)
class RemoveOlder(Deamon):
_remove_older_time = Option('RemoveOlderTime', '23:00')
def __init__(self, bus, app):
self.app = app
Deamon.__init__(self, bus)
@property
def job_execution_time(self):
return self._remove_older_time
def job_run(self):
"""
Execute the job in background.
"""
# Create a generator to loop on repositories.
# Loop on each repos.
for repo in self.app.store.repos():
try:
self._remove_older(repo)
except BaseException:
_logger.exception(
"fail to remove older for user [%r] repo [%r]", repo.owner,
repo)
def _remove_older(self, repo):
"""
Take action to remove older.
"""
assert repo
if repo.keepdays <= 0:
return
# Check history date.
if not repo.last_backup_date:
_logger.info("no backup dates for [%r]", repo.full_path)
return
d = librdiff.RdiffTime() - repo.last_backup_date
d = d.days + repo.keepdays
repo.remove_older(d)
class RdiffwebApp(Application):
"""This class represent the application context."""
_favicon = Option('Favicon',
default=pkg_resources.resource_filename(
'rdiffweb',
'static/favicon.ico')) # @UndefinedVariable
_header_logo = Option('HeaderLogo')
_tempdir = Option('TempDir')
def __init__(self, cfg={}):
self.cfg = {k.lower(): v for k, v in cfg.items()}
# Initialise the template engine.
self.templates = rdw_templating.TemplateManager()
# Get some config
session_path = self.cfg.get("sessiondir", None)
# Initialise the application
config = {
native_str('/'): {
'tools.authform.on':
True,
'tools.i18n.on':
True,
'tools.i18n.default':
'en_US',
'tools.i18n.mo_dir':
pkg_resources.resource_filename(
'rdiffweb', 'locales'), # @UndefinedVariable
'tools.i18n.domain':
'messages',
'tools.encode.on':
True,
'tools.encode.encoding':
'utf-8',
'tools.gzip.on':
True,
'tools.sessions.on':
True,
'tools.proxy.on':
CP_PROXY,
'error_page.default':
self.error_page,
'request.error_response':
self.error_response,
'tools.sessions.storage_type':
'file' if session_path else 'ram',
'tools.sessions.storage_path':
session_path,
},
}
# To work around the new behaviour in CherryPy >= 5.5.0, force usage of
# ISO-8859-1 encoding for URL. This avoid any conversion of the
# URL into UTF-8.
if PY3 and LooseVersion(cherrypy.__version__) >= LooseVersion("5.5.0"):
config[native_str('/')]["request.uri_encoding"] = "ISO-8859-1"
# Initialize the application
Application.__init__(self, root=Root(), config=config)
# Register favicon.ico
self.root.favicon_ico = static(self._favicon)
# Register header_logo
if self._header_logo:
self.root.header_logo = static(self._header_logo)
# Define TEMP env
if self._tempdir:
os.environ["TMPDIR"] = self._tempdir
# create user manager
self.userdb = UserManager(self)
@property
def currentuser(self):
"""
Get the current user.
"""
return cherrypy.serving.request.login
def error_page(self, **kwargs):
"""
Default error page shown to the user when an unexpected error occur.
"""
# Log exception.
logger.exception(kwargs.get('message', ''))
# Check expected response type.
mtype = cherrypy.tools.accept.callable(['text/html', 'text/plain'
]) # @UndefinedVariable
if mtype == 'text/plain':
return kwargs.get('message')
# Try to build a nice error page.
try:
page = Controller()
return page._compile_template('error_page_default.html', **kwargs)
except:
pass
# If failing, send the raw error message.
return kwargs.get('message')
def error_response(self):
"""
Called when ever an exception reach cherrypy core. This implementation
will convert the exception into the right HTTP Error.
"""
code = 500
t = sys.exc_info()[0]
if t == DoesNotExistError:
code = 404
elif t == AccessDeniedError:
code = 403
cherrypy.HTTPError(code).set_response()
@property
def version(self):
"""
Get the current running version (using package info).
"""
# Get version.
try:
return pkg_resources.get_distribution("rdiffweb").version
except:
return "DEV"
@property
def plugins(self):
"""
Return list of plugins.
"""
RiffwebPlugin = namedtuple('RiffwebPlugin', ['name', 'version'])
for group in ['rdiffweb.IUserQuota', 'rdiffweb.IUserChangeListener']:
for e in pkg_resources.iter_entry_points(group):
yield RiffwebPlugin(name=e.name, version=e.dist)
class NotificationPlugin(Deamon, IUserChangeListener):
"""
Send email notification when a repository get too old (without a backup).
"""
_encryption = Option('EmailEncryption', 'none')
_email_host = Option('EmailHost', _get_func=lambda x: x.partition(':')[0])
_email_port = Option('EmailHost',
_get_func=lambda x: int(x.partition(':')[2]))
_email_from = Option('EmailSender')
_email_notification_time = Option('EmailNotificationTime', '23:00')
_smtp_username = Option('EmailUsername', None)
_smtp_password = Option('EmailPassword', None)
_header_name = Option("HeaderName", "rdiffweb")
_send_change_notification = BoolOption("EmailSendChangedNotification",
False)
def __init__(self, bus, app):
self.app = app
self.app.store.add_change_listener(self)
Deamon.__init__(self, bus)
@property
def job_execution_time(self):
return self._email_notification_time
def job_run(self):
"""
Implementation of JobPLugin interface.
Go trough all the repositories and users to send mail in batches.
"""
self.send_notifications()
def user_attr_changed(self, userobj, attrs={}):
"""
Implementation of IUserChangeListener interface.
"""
if not self._send_change_notification:
return
# Leave if the mail was not changed.
if 'email' not in attrs:
return
if not userobj.email:
logging.info("can't sent mail to user [%s] without an email",
userobj.username)
return
# If the email attributes was changed, send a mail notification.
self.send_mail(userobj, _("Email address changed"),
"email_changed.html")
def user_password_changed(self, username, password):
"""
Implementation of IUserChangeListener interface.
"""
# get User object (to get email)
userobj = self.app.store.get_user(username)
assert userobj
if not userobj.email:
logging.info("can't sent mail to user [%s] without an email",
userobj.username)
return
# If the email attributes was changed, send a mail notification.
self.send_mail(userobj, _("Password changed"), "password_changed.html")
def send_notifications(self):
"""
Loop trough all the user repository and send notifications.
"""
now = librdiff.RdiffTime()
def _user_repos():
"""Return a generator trought user repos to be notified."""
for user in self.app.store.users():
# Check if user has email.
if not user.email:
continue
# Identify old repo for current user.
old_repos = []
for repo in user.repo_objs:
# Check if repo has age configured (in days)
maxage = repo.maxage
if not maxage or maxage <= 0:
continue
# Check repo age.
if repo.last_backup_date < (
now - datetime.timedelta(days=maxage)):
old_repos.append(repo)
# Return an item only if user had old repo
if old_repos:
yield user, old_repos
# For each candidate, send mail.
for user, repos in _user_repos():
parms = {'user': user, 'repos': repos}
self.send_mail(user, _('Notification'), 'email_notification.html',
**parms)
def send_mail(self, to_user, subject, template_name, **kwargs):
"""
Reusable method to be called to send email to the user user.
`user` user object where to send the email.
``
"""
# Verify if the users as an email.
assert to_user
assert to_user.email
# Build email from template.
parms = {'user': to_user}
if self._header_name:
parms["header_name"] = self._header_name
parms.update(kwargs)
# Compile both template.
html = self.app.templates.compile_template(template_name, **parms)
text = html2plaintext(html)
# Record the MIME types of both parts - text/plain and text/html.
part1 = MIMEText(text, 'plain', 'utf8')
part2 = MIMEText(html, 'html', 'utf8')
email_from = self._header_name
if self._email_from:
email_from += " <%s>" % self._email_from
msg = MIMEMultipart('alternative')
msg['Subject'] = subject
msg['From'] = email_from
msg['To'] = to_user.email
msg.attach(part1)
msg.attach(part2)
# Open an SMTP connection.
conn = None
if self._encryption == 'ssl':
conn = smtplib.SMTP_SSL(self._email_host, self._email_port)
else:
conn = smtplib.SMTP(self._email_host, self._email_port)
try:
if self._encryption == 'starttls':
conn.starttls()
# Authenticate if required.
if self._smtp_username:
conn.login(self._smtp_username, self._smtp_password)
conn.sendmail(self._email_from, to_user.email, msg.as_string())
finally:
if conn is not None:
conn.quit()
class Store():
"""
This class handle all data storage operations.
"""
_db_file = Option("SQLiteDBFile", "/etc/rdiffweb/rdw.db")
_allow_add_user = BoolOption("AddMissingUser", False)
_admin_user = Option("AdminUser", "admin")
def __init__(self, app):
self.app = app
self._database = SQLiteBackend(self._db_file)
self._password_stores = [LdapPasswordStore(app)]
self._change_listeners = []
# Register entry point.
for entry_point in pkg_resources.iter_entry_points(
'rdiffweb.IUserChangeListener'): # @UndefinedVariable
try:
cls = entry_point.load()
# Creating the listener should register it self.But let make sure of it.
listener = cls(self.app)
if listener not in self._change_listeners:
self._change_listeners.append(listener)
except:
logging.error("IUserChangeListener [%s] fail to load",
entry_point)
def create_admin_user(self):
# Check if admin user exists. If not, created it.
if not self.get_user(self._admin_user):
userobj = self.add_user(self._admin_user, 'admin123')
userobj.is_admin = True
def add_change_listener(self, listener):
self._change_listeners.append(listener)
def remove_change_listener(self, listener):
self._change_listeners.remove(listener)
def add_user(self, user, password=None, attrs=None):
"""
Used to add a new user with an optional password.
"""
assert password is None or isinstance(password, str)
# Check if user already exists.
if self.get_user(user):
raise RdiffError(_("User %s already exists." % (user, )))
# Find a database where to add the user
logger.debug("adding new user [%s]", user)
if password:
inserted = self._database.insert('users',
username=user,
password=_hash_password(password))
else:
inserted = self._database.insert('users',
username=user,
password='')
assert inserted
record = self._database.findone('users', username=user)
userobj = UserObject(self, record)
self._notify('user_added', userobj, attrs)
# Return user object
return userobj
def count_users(self):
return self._database.count('users')
def count_repos(self):
return self._database.count('repos')
def get_user(self, user):
"""Return a user object."""
record = self._database.findone('users', username=user)
if record:
return UserObject(self, record)
return None
def users(self, search=None, criteria=None):
"""
Search users database. Return a generator of user object.
search: Define a search term to look into email or username.
criteria: Define a search filter: admins, ldap
"""
if search:
users = self._database.search('users', search, 'username',
'useremail')
elif criteria:
if criteria == 'admins':
users = self._database.find('users', isadmin=1)
elif criteria == 'ldap':
users = self._database.find('users', password='')
else:
return
else:
users = self._database.find('users')
for record in users:
yield UserObject(self, record)
def repos(self, search=None, criteria=None):
"""
Quick listing of all the repository object for all user.
search: Define a search term to look into path, email or username.
criteria: Define a search filter: ok, failed, interrupted, in_progress
"""
if search:
repos = self._database.search('repos', search, 'RepoPath',
'username', 'useremail')
else:
repos = self._database.find('repos')
for record in repos:
user_record = self._database.findone('users',
userid=record['userid'])
user_obj = UserObject(self, user_record)
repo_obj = RepoObject(user_obj, record)
if not criteria or criteria == repo_obj.status[0]:
yield repo_obj
def login(self, user, password):
"""
Called to authenticate the given user.
Check if the credentials are valid. Then may actually add the user
in database if allowed.
If valid, return the username. Return False if the user exists but the
password doesn't matches. Return None if the user was not found in any
password store.
The return user object. The username may not be equals to the given username.
"""
assert isinstance(user, str)
assert password is None or isinstance(user, str)
# Validate credential using database first.
logger.debug("validating user [%s] credentials", user)
record = self._database.findone('users', username=user)
if record and record['password'] == _hash_password(password):
userobj = UserObject(self, record)
self._notify('user_logined', userobj, None)
return userobj
# Fallback to LDAP
for store in self._password_stores:
try:
valid = store.are_valid_credentials(user, password)
if valid:
break
except:
pass
if not valid:
return None
# Get real username and external attributes.
real_user, attrs = valid
# Check if user exists in database
userobj = self.get_user(real_user)
if not userobj:
if self._allow_add_user:
# Create user
userobj = self.add_user(real_user, attrs=attrs)
else:
logger.info("user [%s] not found in database", real_user)
return None
self._notify('user_logined', userobj, attrs)
return userobj
def _notify(self, mod, *args):
for listener in self._change_listeners:
# Support divergent account change listener implementations too.
try:
logger.debug('notify %s#%s()', listener.__class__.__name__,
mod)
getattr(listener, mod)(*args)
except:
logger.warning('IUserChangeListener [%s] fail to run [%s]',
listener.__class__.__name__,
mod,
exc_info=1)
class AdminPage(Controller):
"""Administration pages. Allow to manage users database."""
logfile = Option('logfile')
logaccessfile = Option('logaccessfile')
def _check_user_root_dir(self, directory):
"""Raised an exception if the directory is not valid."""
if not os.access(directory, os.F_OK) or not os.path.isdir(directory):
raise RdiffWarning(
_("User root directory %s is not accessible!") % directory)
def _get_log_files(self):
"""
Return a list of log files to be shown in admin area.
"""
logfiles = [self.logfile, self.logaccessfile]
logfiles = [fn for fn in logfiles if fn]
return [os.path.basename(fn) for fn in logfiles]
def _get_log_data(self, logfile, num=2000):
"""
Return a list of log files to be shown in admin area.
"""
logfiles = [self.logfile, self.logaccessfile]
logfiles = [fn for fn in logfiles if fn]
for fn in logfiles:
if logfile == os.path.basename(fn):
try:
return subprocess.check_output(
['tail', '-n', str(num), fn],
stderr=subprocess.STDOUT).decode('utf-8')
except:
logging.exception('fail to get log file content')
return "Error getting file content"
@cherrypy.expose
def default(self):
params = {
"user_count": self.app.store.count_users(),
"repo_count": self.app.store.count_repos()
}
return self._compile_template("admin.html", **params)
@cherrypy.expose
def logs(self, filename=u""):
# Check if the filename is valid.
data = ""
logfiles = self._get_log_files()
if logfiles:
if not filename:
filename = logfiles[0]
if filename not in logfiles:
raise cherrypy.HTTPError(404)
data = self._get_log_data(filename)
params = {
"filename": filename,
"logfiles": logfiles,
"data": data,
}
return self._compile_template("admin_logs.html", **params)
@cherrypy.expose
def users(self,
criteria=u"",
search=u"",
action=u"",
username=u"",
email=u"",
password=u"",
user_root=u"",
is_admin=u""):
# If we're just showing the initial page, just do that
params = {}
if self._is_submit():
try:
params = self._users_handle_action(action, username, email,
password, user_root,
is_admin)
except RdiffWarning as e:
params['warning'] = str(e)
except RdiffError as e:
params['error'] = str(e)
params.update({
"criteria":
criteria,
"search":
search,
"users":
list(self.app.store.users(search=search, criteria=criteria))
})
# Build users page
return self._compile_template("admin_users.html", **params)
@cherrypy.expose
def repos(self, criteria=u"", search=u""):
params = {
"criteria": criteria,
"search": search,
"repos":
list(self.app.store.repos(search=search, criteria=criteria))
}
return self._compile_template("admin_repos.html", **params)
@cherrypy.expose
def sysinfo(self):
params = {
"version": self.app.version,
"plugins": self.app.plugins,
# Config
"cfg": {
k: '********' if 'password' in k else v
for k, v in self.app.cfg.items()
},
# System Info entries
"pyinfo": list(get_pyinfo()),
"osinfo": list(get_osinfo()),
"hwinfo": list(get_hwinfo()),
"ldapinfo": list(get_pkginfo()),
}
return self._compile_template("admin_sysinfo.html", **params)
def _users_handle_action(self, action, username, email, password,
user_root, is_admin):
success = ""
# We need to change values. Change them, then give back that main
# page again, with a message
if username == self.app.currentuser.username:
# Don't allow the user to changes it's "admin" state.
is_admin = self.app.currentuser.is_admin
is_admin = str(is_admin).lower() in ['on', 'true', '1']
# Fork the behaviour according to the action.
if action == "edit":
user = self.app.store.get_user(username)
logger.info("updating user [%s] info", user)
if password:
user.set_password(password, old_password=None)
user.user_root = user_root
user.is_admin = is_admin
# Avoid updating the email fields is it didn'T changed. see pdsl/minarca#187
if email != user.email:
user.email = email
success = _("User information modified successfully.")
# Check and update user directory
if user.user_root:
self._check_user_root_dir(user.user_root)
rdw_spider_repos.find_repos_for_user(user)
elif action == "add":
if username == "":
raise RdiffWarning(_("The username is invalid."))
logger.info("adding user [%s]", username)
user = self.app.store.add_user(username, password)
if user_root:
user.user_root = user_root
user.is_admin = is_admin
user.email = email
# Check and update user directory
if user.user_root:
self._check_user_root_dir(user.user_root)
rdw_spider_repos.find_repos_for_user(user)
success = _("User added successfully.")
if action == "delete":
if username == self.app.currentuser.username:
raise RdiffWarning(_("You cannot remove your own account!"))
user = self.app.store.get_user(username)
if not user:
raise RdiffWarning(_("User doesn't exists!"))
user.delete()
success = _("User account removed.")
# Return messages
return {'success': success}
class Store():
"""
This class handle all data storage operations.
"""
_db_file = Option("SQLiteDBFile", "/etc/rdiffweb/rdw.db")
_allow_add_user = BoolOption("AddMissingUser", False)
_admin_user = Option("AdminUser", "admin")
def __init__(self, app):
self.app = app
from rdiffweb.core.store_sqlite import SQLiteBackend
self._database = SQLiteBackend(self._db_file)
self._password_stores = [LdapPasswordStore(app)]
self._change_listeners = []
# Register entry point.
for entry_point in pkg_resources.iter_entry_points(
'rdiffweb.IUserChangeListener'): # @UndefinedVariable
try:
cls = entry_point.load()
# Creating the listener should register it self.But let make sure of it.
listener = cls(self.app)
if listener not in self._change_listeners:
self._change_listeners.append(listener)
except:
logging.error("IUserChangeListener [%s] fail to load",
entry_point)
def create_admin_user(self):
# Check if admin user exists. If not, created it.
if not self.get_user(self._admin_user):
userobj = self.add_user(self._admin_user, 'admin123')
userobj.role = ADMIN_ROLE
def add_change_listener(self, listener):
self._change_listeners.append(listener)
def remove_change_listener(self, listener):
self._change_listeners.remove(listener)
def add_user(self, user, password=None, attrs=None):
"""
Used to add a new user with an optional password.
"""
assert password is None or isinstance(password, str)
# Check if user already exists.
if self.get_user(user):
raise RdiffError(_("User %s already exists." % (user, )))
# Find a database where to add the user
logger.debug("adding new user [%s]", user)
if password:
inserted = self._database.insert('users',
username=user,
password=hash_password(password))
else:
inserted = self._database.insert('users',
username=user,
password='')
assert inserted
record = self._database.findone('users', username=user)
userobj = UserObject(self, record)
self._notify('user_added', userobj, attrs)
# Return user object
return userobj
def count_users(self):
return self._database.count('users')
def count_repos(self):
return self._database.count('repos')
def get_repo(self, name, as_user=None):
"""
Return the repository identified as `name`.
`name` should be <username>/<repopath>
"""
username, repopath = _split_path(name)
repopath = decodefilename(repopath)
# Check permissions
as_user = as_user or self.app.currentuser
assert as_user, "as_user or current user must be defined"
if username != as_user.username and not as_user.is_admin:
raise AccessDeniedError(name)
# Get the userid associated to the username.
user_obj = self.get_user(username)
if not user_obj:
raise DoesNotExistError(name)
# Get the repo object.
return user_obj.get_repo(repopath)
def get_repo_path(self, path, as_user=None):
"""
Return a the repository identified by the given `path`.
`path` should be <username>/<repopath>/<subdir>
"""
assert isinstance(path, bytes) or isinstance(path, str)
sep = b'/' if isinstance(path, bytes) else '/'
path = path.strip(sep) + sep
# Since we don't know which part of the "path" is the repopath,
# we need to do multiple search.
try:
startpos = 0
while True:
pos = path.index(sep, startpos)
try:
repo_obj = self.get_repo(path[:pos], as_user)
path_obj = repo_obj.get_path(path[pos + 1:])
return repo_obj, path_obj
except DoesNotExistError:
# continue looping
startpos = pos + 1
except ValueError:
raise DoesNotExistError(path)
def get_user(self, user):
"""Return a user object."""
record = self._database.findone('users', username=user)
if record:
return UserObject(self, record)
return None
def users(self, search=None, criteria=None):
"""
Search users database. Return a generator of user object.
search: Define a search term to look into email or username.
criteria: Define a search filter: admins, ldap
"""
if search:
users = self._database.search('users', search, 'username',
'useremail')
elif criteria:
if criteria == 'admins':
users = self._database.find('users', role=ADMIN_ROLE)
elif criteria == 'ldap':
users = self._database.find('users', password='')
else:
return
else:
users = self._database.find('users')
for record in users:
yield UserObject(self, record)
def repos(self, search=None, criteria=None):
"""
Quick listing of all the repository object for all user.
search: Define a search term to look into path, email or username.
criteria: Define a search filter: ok, failed, interrupted, in_progress
"""
if search:
repos = self._database.search('repos', search, 'RepoPath',
'username', 'useremail')
else:
repos = self._database.find('repos')
for record in repos:
user_record = self._database.findone('users',
userid=record['userid'])
user_obj = UserObject(self, user_record)
repo_obj = RepoObject(user_obj, record)
if not criteria or criteria == repo_obj.status[0]:
yield repo_obj
def login(self, user, password):
"""
Called to authenticate the given user.
Check if the credentials are valid. Then may actually add the user
in database if allowed.
If valid, return the username. Return False if the user exists but the
password doesn't matches. Return None if the user was not found in any
password store.
The return user object. The username may not be equals to the given username.
"""
assert isinstance(user, str)
assert password is None or isinstance(user, str)
# Validate credential using database first.
logger.debug("validating user [%s] credentials", user)
userobj = self.get_user(user)
if userobj and userobj.hash_password:
if not check_password(password, userobj.hash_password):
return None
self._notify('user_logined', userobj, None)
return userobj
# Fallback to LDAP
if userobj or self._allow_add_user:
for store in self._password_stores:
try:
valid = store.are_valid_credentials(user, password)
if valid:
real_user, attrs = valid
if not userobj:
userobj = self.add_user(real_user, attrs=attrs)
self._notify('user_logined', userobj, attrs)
return userobj
except:
logger.warn('fail to validate credentials', exc_info=1)
return None
def _notify(self, mod, *args):
for listener in self._change_listeners:
# Support divergent account change listener implementations too.
try:
logger.debug('notify %s#%s()', listener.__class__.__name__,
mod)
getattr(listener, mod)(*args)
except:
logger.warning('IUserChangeListener [%s] fail to run [%s]',
listener.__class__.__name__,
mod,
exc_info=1)
class UserManager():
"""
This class handle all user operation. This class is greatly inspired from
TRAC account manager class.
"""
_db_file = Option("SQLiteDBFile", "/etc/rdiffweb/rdw.db")
_allow_add_user = BoolOption("AddMissingUser", False)
_admin_user = Option("AdminUser", "admin")
def __init__(self, app):
self.app = app
self._database = SQLiteUserDB(self._db_file)
self._password_stores = [LdapPasswordStore(app)]
self._change_listeners = []
# Register entry point.
for entry_point in pkg_resources.iter_entry_points(
'rdiffweb.IUserChangeListener'): # @UndefinedVariable
try:
cls = entry_point.load()
# Creating the listener should register it self.But let make sure of it.
listener = cls(self.app)
if listener not in self._change_listeners:
self._change_listeners.append(listener)
except:
logging.error("IUserChangeListener [%s] fail to load",
entry_point)
# Check if admin user exists. If not, created it.
if not self.exists(self._admin_user):
userobj = self.add_user(self._admin_user, 'admin123')
userobj.is_admin = True
def add_change_listener(self, listener):
self._change_listeners.append(listener)
def remove_change_listener(self, listener):
self._change_listeners.remove(listener)
def add_user(self, user, password=None, attrs=None):
"""
Used to add a new user with an optional password.
"""
assert password is None or isinstance(password, str)
# Check if user already exists.
if self._database.exists(user):
raise RdiffError(_("User %s already exists." % (user, )))
# Find a database where to add the user
logger.debug("adding new user [%s]", user)
self._database.add_user(user, password)
userobj = UserObject(self, self._database, user)
self._notify('user_added', userobj, attrs)
# Return user object
return userobj
def delete_user(self, user):
"""
Delete the given user from password store.
Return True if the user was deleted. Return False if the user didn't
exists.
"""
if hasattr(user, 'username'):
user = user.username
if user == self._admin_user:
raise ValueError(_("can't delete admin user"))
# Delete user from database (required).
logger.info("deleting user [%s] from database", user)
self._database.delete_user(user)
self._notify('user_deleted', user)
return True
def exists(self, user):
"""
Verify if the given user exists in our database.
Return True if the user exists. False otherwise.
"""
return self._database.exists(user)
def get_user(self, user):
"""Return a user object."""
if not self.exists(user):
return None
return UserObject(self, self._database, user)
def users(self, search=None, criteria=None):
"""
Search users database. Return a generator of user object.
search: Define a search term to look into email or username.
criteria: Define a search filter: admins, ldap
"""
# TODO Add criteria as required.
for username in self._database.users(search, criteria):
yield UserObject(self, self._database, username)
def repos(self, search=None, criteria=None):
"""
Quick listing of all the repository object for all user.
search: Define a search term to look into path, email or username.
criteria: Define a search filter: ok, failed, interrupted, in_progress
"""
for username, repo in self._database.repos(search, criteria):
user_obj = UserObject(self, self._database, username)
repo_obj = RepoObject(user_obj, repo)
if not criteria or criteria == repo_obj.status[0]:
yield repo_obj
def login(self, user, password):
"""
Called to authenticate the given user.
Check if the credentials are valid. Then may actually add the user
in database if allowed.
If valid, return the username. Return False if the user exists but the
password doesn't matches. Return None if the user was not found in any
password store.
The return user object. The username may not be equals to the given username.
"""
assert isinstance(user, str)
assert password is None or isinstance(user, str)
# Validate the credentials
logger.debug("validating user [%s] credentials", user)
for store in [self._database] + self._password_stores:
try:
valid = store.are_valid_credentials(user, password)
if valid:
break
except:
pass
if not valid:
return None
# Get real username and external attributes.
# Some password store provide extra attributes.
if isinstance(valid, str):
real_user, attrs = valid, None
else:
real_user, attrs = valid
# Check if user exists in database
userobj = self.get_user(real_user)
if not userobj:
if self._allow_add_user:
# Create user
userobj = self.add_user(real_user, attrs=attrs)
else:
logger.info("user [%s] not found in database", real_user)
return None
self._notify('user_logined', userobj, attrs)
return userobj
def set_password(self, username, password, old_password=None):
# Try to update the user password.
for store in self._password_stores:
try:
valid = store.are_valid_credentials(username, old_password)
if valid:
store.set_password(username, password, old_password)
return
except:
pass
# Fallback to database
self._database.set_password(username, password, old_password)
self._notify('user_password_changed', username, password)
def _notify(self, mod, *args):
for listener in self._change_listeners:
# Support divergent account change listener implementations too.
try:
logger.debug('notify %s#%s()', listener.__class__.__name__,
mod)
getattr(listener, mod)(*args)
except:
logger.warning('IUserChangeListener [%s] fail to run [%s]',
listener.__class__.__name__,
mod,
exc_info=1)
class MinarcaUserSetup(IUserChangeListener, IUserQuota):
"""
This plugin provide feedback information to the users about the disk usage.
Since we define quota, this plugin display the user's quota.
"""
_quota_api_url = Option('MinarcaQuotaApiUrl', 'http://*****:*****@localhost:8081/')
_mode = IntOption('MinarcaUserDirMode', 0o0770)
_owner = Option('MinarcaUserDirOwner', 'minarca')
_group = Option('MinarcaUserDirGroup', 'minarca')
_basedir = Option('MinarcaUserBaseDir', default='/backups/')
_minarca_shell = Option('MinarcaShell', default='/opt/minarca/bin/minarca-shell')
_auth_options = Option('MinarcaAuthOptions', default='no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty')
_minarca_remotehost = Option('MinarcaRemoteHost')
_minarca_identity = Option('MinarcaRemoteHostIdentity', default='/etc/ssh')
_restrited_basedir = BoolOption('MinarcaRestrictedToBasedDir', default=True)
_redirect_help = Option('MinarcaHelpURL', default='https://www.ikus-soft.com/en/support/#form')
def __init__(self, app):
self.app = app
self.app.root.api.minarca = self.get_minarca
self.app.root.help = self.get_help
self.session = requests.Session()
self.session.mount('https://', TimeoutHTTPAdapter(pool_connections=2, pool_maxsize=5))
self.session.mount('http://', TimeoutHTTPAdapter(pool_connections=2, pool_maxsize=5))
@cherrypy.expose
@cherrypy.config(**{'tools.authform.on': False, 'tools.i18n.on': False, 'tools.authbasic.on': False, 'tools.sessions.on': False, 'error_page.default': False})
def get_help(self):
raise cherrypy.HTTPRedirect(self._redirect_help)
@cherrypy.expose
@cherrypy.tools.json_out()
def get_minarca(self):
# RemoteHost
remotehost = self._minarca_remotehost
if not remotehost:
remotehost = urlparse(cherrypy.request.base).hostname
# Identity known_hosts
identity = ""
files = [f for f in os.listdir(self._minarca_identity) if f.startswith('ssh_host') if f.endswith('.pub')]
for fn in files:
with open(os.path.join(self._minarca_identity, fn)) as fh:
if ':' in remotehost:
hostname, port = remotehost.split(':', 1)
identity += "[" + hostname + "]:" + port + " " + fh.read()
else:
identity += remotehost + " " + fh.read()
# Get remote host value from config or from URL
return {
"version": pkg_resources.get_distribution("minarca-server").version,
"remotehost": remotehost,
"identity": identity,
}
def get_disk_usage(self, userobj):
"""
Return the user disk space.
"""
assert isinstance(userobj, UserObject)
# Get Quota from web service
url = os.path.join(self._quota_api_url, 'quota', userobj.username)
r = self.session.get(url, timeout=1)
r.raise_for_status()
diskspace = r.json()
assert diskspace and isinstance(diskspace, dict) and 'avail' in diskspace and 'used' in diskspace and 'size' in diskspace
return diskspace
def get_disk_quota(self, userobj):
"""
Get's user's disk quota.
"""
return self.get_disk_usage(userobj)['size']
def set_disk_quota(self, userobj, quota):
"""
Sets the user's quota.
"""
assert isinstance(userobj, UserObject)
assert quota
# Always update unless quota not define
logger.info('set user [%s] quota [%s]', userobj.username, quota)
url = os.path.join(self._quota_api_url, 'quota', userobj.username)
r = self.session.post(url, data={'size': quota}, timeout=1)
r.raise_for_status()
diskspace = r.json()
assert diskspace and isinstance(diskspace, dict) and 'avail' in diskspace and 'used' in diskspace and 'size' in diskspace
return diskspace
def user_logined(self, userobj, attrs):
"""
Need to verify LDAP quota and update ZFS quota if required.
"""
assert isinstance(userobj, UserObject)
# TODO This is specific to Minarca Saas. We need to change this.
# Get quota value from LDAP field.
quota = False
descriptions = attrs and attrs.get('description')
if descriptions:
quota_gb = [
int(x[1:]) for x in descriptions
if x.startswith(b"v") and x[1:].isdigit()]
if quota_gb:
quota_gb = max(quota_gb)
quota = quota_gb * 1024 * 1024 * 1024
# If we found a quota value, use quota api to set it.
logger.info('found user [%s] quota [%s] from attrs', userobj.username, quota)
if quota:
userobj.disk_quota = quota
def user_added(self, userobj, attrs):
"""
When added (manually or not). Try to get data from LDAP.
"""
assert isinstance(userobj, UserObject)
try:
self._update_user_email(userobj, attrs)
except:
logger.warning('fail to update user [%s] email from LDAP', userobj.username, exc_info=1)
try:
self._update_user_root(userobj, attrs)
except:
logger.warning('fail to update user [%s] root', userobj.username, exc_info=1)
def user_attr_changed(self, userobj, attrs={}):
"""
Listen to users attributes change to update the minarca authorized_keys.
"""
if 'user_root' in attrs:
self._update_user_root(userobj, attrs)
# Update minarca's authorized_keys when users update their ssh keys.
if 'authorizedkeys' in attrs or 'user_root' in attrs:
# TODO schedule a background task to update the authorized_keys.
self._update_authorized_keys()
def user_deleted(self, username):
"""
When user get dleted, update the authorized_key.
"""
self._update_authorized_keys()
def _update_user_email(self, userobj, attrs):
"""
Called to update the user email and home directory from LDAP info.
"""
# Get user email from LDAP
mail = attrs and attrs.get('mail', None)
if not mail:
return
# mail might be a list.
if hasattr(mail, '__getitem__') and len(mail):
mail = mail[0]
if isinstance(mail, bytes):
mail = mail.decode('utf-8')
logger.info('update user [%s] email from LDAP [%s]', userobj.username, mail)
userobj.email = mail
def _update_user_root(self, userobj, attrs):
"""
Called to update the user's home directory. Either to define it with
default value or restrict it to base dir.
"""
# Define default user_home if not provided
user_root = userobj.user_root or os.path.join(self._basedir, userobj.username)
# Normalise path to avoid relative path.
user_root = os.path.abspath(user_root)
# Verify if the user_root is inside base dir.
if self._restrited_basedir and not user_root.startswith(self._basedir):
logger.warn('restrict user [%s] root [%s] to base dir [%s]', userobj.username, user_root, self._basedir)
user_root = os.path.join(self._basedir, userobj.username)
# Persist the value if different then original
if userobj.user_root != user_root:
userobj.user_root = user_root
# Create folder if inside our base dir and missing.
if user_root.startswith(self._basedir) and not os.path.exists(user_root):
logger.info('creating user [%s] root dir [%s]', userobj.username, user_root)
try:
os.mkdir(user_root)
# Change mode
os.chmod(user_root, self._mode)
# Change owner
os.chown(user_root, pwd.getpwnam(self._owner).pw_uid, grp.getgrnam(self._group).gr_gid)
except:
logger.exception('fail to create user [%s] root dir [%s]', userobj.username, user_root)
def _update_authorized_keys(self):
"""
Used to update the authorized_keys of minarca user.
"""
# Create ssh subfolder
ssh_dir = os.path.join(self._basedir, '.ssh')
if not os.path.exists(ssh_dir):
logger.info("creating .ssh folder [%s]", ssh_dir)
os.mkdir(ssh_dir, 0o700)
try:
os.chown(ssh_dir, pwd.getpwnam(self._owner).pw_uid, grp.getgrnam(self._group).gr_gid)
except:
logger.warn("fail to set permissions on [%s] folder", ssh_dir)
# Create the authorized_keys file
filename = os.path.join(ssh_dir, 'authorized_keys')
if not os.path.exists(filename):
logger.info("creating authorized_keys [%s]", filename)
with open(filename, 'w+'):
os.utime(filename, None)
# change file permissions
os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR)
val = os.stat(ssh_dir)
# Also try to change owner
os.chown(filename, val.st_uid, val.st_gid)
# Get list of keys
seen = set()
new_data = StringIO()
for userobj in self.app.store.users():
for key in userobj.authorizedkeys:
if key.fingerprint in seen:
logger.warn("duplicates key %s, sshd will ignore it")
else:
seen.add(key.fingerprint)
# Add option to the key
options = """command="%s '%s' '%s'",%s""" % (self._minarca_shell, userobj.username, userobj.user_root, self._auth_options)
key = AuthorizedKey(options=options, keytype=key.keytype, key=key.key, comment=key.comment)
# Write the new key
authorizedkeys.add(new_data, key)
# Write the new file
logger.info("updating authorized_keys file [%s]", filename)
with open(filename, 'w', encoding='utf-8') as f:
f.write(new_data.getvalue())
class LdapPasswordStore():
"""Wrapper for LDAP authentication.
This implementation assume the LDAP is using the system encoding."""
uri = Option("LdapUri", doc="Get Ldap URI")
base_dn = Option("LdapBaseDn", "", doc="Get Base DN")
scope = Option("LdapScope", "subtree")
tls = BoolOption("LdapTls", "false", doc="Check if TLs is enabled")
attribute = Option("LdapAttribute", "uid", doc="Get attribute")
filter = Option("LdapFilter", "(objectClass=*)")
require_group = Option("LdapRequiredGroup")
group_attribute = Option("LDAPGroupAttribute", "member")
group_attribute_is_dn = BoolOption("LDAPGroupAttributeIsDN", "true")
bind_dn = Option("LdapBindDn", "")
bind_password = Option("LdapBindPassword", "")
version = IntOption("LdapVersion", "3")
network_timeout = IntOption("LdapNetworkTimeout", "100")
timeout = IntOption("LdapTimeout", "300")
encoding = Option("LdapEncoding", "utf-8", doc="Get default LdapEncoding")
allow_password_change = BoolOption(
"LdapAllowPasswordChange",
"false",
doc="Check if password change are allowed.")
check_shadow_expire = BoolOption(
"LdapCheckShadowExpire",
"false",
doc="Enable verification of Shadow Expire.")
def __init__(self, app):
self.app = app
def are_valid_credentials(self, username, password):
"""Check if the given credential as valid according to LDAP."""
assert isinstance(username, str)
assert isinstance(password, str)
# Skip validation if LdapUri is not provided
if not self.uri:
return False
def check_crendential(l, r):
# Check results
if len(r) != 1:
logger.debug("user [%s] not found in LDAP", username)
return None
# Bind using the user credentials. Throws an exception in case of
# error.
l.simple_bind_s(r[0][0], password)
try:
logger.info("user [%s] found in LDAP", username)
# Verify the shadow expire
if self.check_shadow_expire:
shadow_expire = self._attr_shadow_expire(r)
# Convert nb. days into seconds.
if shadow_expire and shadow_expire * 24 * 60 * 60 < time.time(
):
logger.warn("user account %s expired: %s", username,
shadow_expire)
raise RdiffError(
_('User account %s expired.' % username))
# Get username
dn = r[0][0]
new_username = self._decode(r[0][1][self.attribute][0])
# Verify if the user is member of the required group
if self.require_group:
value = dn if self.group_attribute_is_dn else new_username
logger.info("check if user [%s] is member of [%s]", value,
self.require_group)
if not l.compare_s(self.require_group,
self.group_attribute, value):
raise RdiffError(
_('Permissions denied for user account %s.' %
username))
finally:
l.unbind_s()
# Return the username
return new_username, r[0][1]
# Execute the LDAP operation
try:
return self._execute(username, check_crendential)
except:
logger.exception("can't validate user [%s] credentials", username)
return False
def _attr(self, r, attr):
if isinstance(attr, list):
return dict([(x, r[0][1][x]) for x in attr if x in r[0][1]])
elif attr in r[0][1]:
if isinstance(r[0][1][attr], list):
return [self._decode(x) for x in r[0][1][attr]]
else:
return self._decode(r[0][1][attr])
return None
def _attr_shadow_expire(self, r):
"""Get Shadow Expire value from `r`."""
# get Shadow expire value
shadow_expire = self._attr(r, 'shadowExpire')
if not shadow_expire:
return None
if isinstance(shadow_expire, list):
shadow_expire = shadow_expire[0]
return int(shadow_expire)
def _decode(self, value):
"""If required, decode the given bytes str into unicode."""
if isinstance(value, bytes):
value = value.decode(encoding=self.encoding)
return value
def _execute(self, username, function):
assert isinstance(username, str)
"""Reusable method to run LDAP operation."""
assert self.uri, "LdapUri must be define in configuration"
assert self.base_dn, "LdapBaseDn must be define in configuration"
if self.scope == "base":
scope = ldap.SCOPE_BASE
elif self.scope == "onelevel":
scope = ldap.SCOPE_ONELEVEL
else:
scope = ldap.SCOPE_SUBTREE
# try STARTLS if configured
if self.tls:
ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
# Check LDAP credential only.
l = ldap.initialize(self.uri)
# Set v2 or v3
if self.version == 2:
l.protocol_version = ldap.VERSION2
else:
l.protocol_version = ldap.VERSION3
try:
# Bind to the LDAP server
logger.debug("binding to ldap server {}".format(self.uri))
l.simple_bind_s(self.bind_dn, self.bind_password)
# Search the LDAP server
search_filter = "(&{}({}={}))".format(self.filter, self.attribute,
username)
logger.debug("search ldap server: {}/{}?{}?{}?{}".format(
self.uri, self.base_dn, self.attribute, scope, search_filter))
r = l.search_s(self.base_dn, scope, search_filter)
# Execute operation
return function(l, r)
except ldap.LDAPError as e:
l.unbind_s()
# Handle the LDAP exception and build a nice user message.
logger.warning('ldap error', exc_info=1)
msg = _("An LDAP error occurred: %s")
ldap_msg = str(e)
if hasattr(e, 'message') and isinstance(e.message, dict):
if 'desc' in e.message:
ldap_msg = e.message['desc']
if 'info' in e.message:
ldap_msg = e.message['info']
raise RdiffError(msg % ldap_msg)
def exists(self, username):
"""Check if the user exists in LDAP"""
def check_user_exists(l, r): # @UnusedVariable
# Check the results
if len(r) != 1:
logger.debug("user [%s] not found", username)
return False
logger.debug("user [%s] found", username)
return True
# Execute the LDAP operation
return self._execute(username, check_user_exists)
def set_password(self, username, password, old_password=None):
"""Update the password of the given user."""
assert isinstance(username, str)
assert old_password is None or isinstance(old_password, str)
assert isinstance(password, str)
# Do nothing if password is empty
if not password:
raise RdiffError(_("Password can't be empty."))
# Check if users are allowed to change their password in LDAP.
if not self.allow_password_change:
logger.warn(
"authentication backend for user [%s] does not support changing the password",
username)
raise RdiffError(
_("LDAP users are not allowed to change their password."))
# Check if old_password id valid
if old_password and not self.are_valid_credentials(
username, old_password):
raise RdiffError(_("Wrong password."))
# Update the username password of the given user. If possible.
return self._set_password_in_ldap(username, old_password, password)
def _set_password_in_ldap(self, username, old_password, password):
def change_passwd(l, r):
if len(r) != 1:
raise RdiffError(_("User %s not found." % (username, )))
# Bind using the user credentials. Throws an exception in case of
# error.
if old_password is not None:
l.simple_bind_s(r[0][0], old_password)
l.passwd_s(r[0][0], old_password, password)
l.unbind_s()
logger.info("password for user [%s] is updated in LDAP", username)
# User updated, return False
return False
# Execute the LDAP operation
logger.debug("updating password for [%s] in LDAP", username)
return self._execute(username, change_passwd)