def loop(self):
conn = Connection()
logger.debug("Starting Scheduler Loop")
# Initialize Heartbeat
if self.heartbeat > 0:
next_hb = datetime.now() + timedelta(seconds=self.heartbeat)
while self.running:
for job_id, job in self.jobs.items():
# Nextcall overpassed, execute job
now = datetime.now()
if now > job["nextcall"]:
self.run(job)
delta_kwarg = {job["interval_type"]: job["interval"]}
# TODO: Reusing 'now' reduces time drifting, but
# will make the job fall behind the nextcall if it
# takes longer than its delta, and therefore the app
# won't run it again until restart.
new_nextcall = now + relativedelta(**delta_kwarg)
# Update Internal Value
self.jobs[job_id]["nextcall"] = new_nextcall
# Update Database Entry
conn.db["base.cron"].update_one(
{"_id": job_id}, {"$set": {
"nextcall": new_nextcall
}})
# Handle Heartbeat
if self.heartbeat > 0:
if datetime.now() > next_hb:
logger.info("Scheduler Heartbeat")
next_hb = datetime.now() + timedelta(
seconds=self.heartbeat)
time.sleep(1)
logger.debug("Scheduler Loop has been terminated")
def handle_call(data, uid):
"""
Take an action according to params values
TODO: This is a provisory method until
there's something more sophisticated.
"""
p = data["params"]
method = p["method"]
cls = registry[p["model"]]
conn = Connection()
client = conn.cl
if config.DB_REPLICASET_ENABLE:
with client.start_session() as session:
with session.start_transaction():
env = Environment(uid, session)
model = cls(env)
result = call_method(p, model, method)
else:
env = Environment(uid)
model = cls(env)
result = call_method(p, model, method)
return result
def ensure_root_user():
""" Create root user if it doesn't exist,
or ensure its password matches the one
specified through the environment variables.
"""
from olaf.tools import config
from olaf.db import Connection
from bson import ObjectId
from werkzeug.security import generate_password_hash
# Generate hashed password
passwd = generate_password_hash(config.ROOT_PASSWORD)
conn = Connection()
root = conn.db["base.user"].find_one({"_id": root_uid})
if not root:
# Create root user
logger.warning("Root user is not present, creating...")
conn.db["base.user"].insert_one({
"_id": root_uid,
"name": "root",
"email": "root",
"password": passwd
})
else:
# Update root user's password
logger.info("Overwriting root user password")
conn.db["base.user"].update_one(
{"_id": root_uid},
{"$set": {
"name": "root",
"email": "root",
"password": passwd
}})
def __init__(self, uid, session=None, context=dict()):
_context = {"uid": uid}
for key, value in context.items():
_context[key] = value
self.context = frozendict(_context)
self.session = session
self.registry = registry
self.conn = Connection()
self.cache = DocumentCache(session)
def start_session():
conn = Connection()
client = conn.cl
with client.start_session() as session:
with session.start_transaction():
uid = ObjectId("000000000000000000000000")
env = Environment(uid, session)
baseUser = registry["base.user"]
rootUser = baseUser(env)
yield rootUser
session.abort_transaction()
def run(self, job):
logger.info("Running job {} ({})...".format(job["_id"], job["name"]))
# Generate context variables
conn = Connection()
client = conn.cl
with client.start_session() as session:
with session.start_transaction():
env = Environment(job["_id"], session)
_locals = {**LOCALS, "env": env}
safe_eval(job["code"], _locals)
return
def check_access(model_name, operation, uid):
"""
Check if a given user can perform
a given operation on a given model
"""
# Root user bypasses all security checks
if uid == ObjectId("000000000000000000000000"):
return
conn = Connection()
user = conn.db["base.user"].find_one({"_id": uid})
if not user:
raise AccessError("User not found")
# Search in user/group many2many intermediate collection
# for groups this is user is related to.
user_group_rels = conn.db["base.user.group.rel"].find({"user_oid": uid})
# Create a list of groups this user belongs to
groups = [rel["group_oid"] for rel in user_group_rels]
# Abort right here if user doesn't belong to any groups
if not groups:
raise AccessError("Access Denied -- Model: '{}' "
"Operation: '{}' - User: '******'".format(
model_name, operation, user["_id"]))
# Search for all ACLs associated to all this groups
acls = conn.db["base.model.access"].find({
"group_id": {
"$in": groups
},
"model": model_name
})
# Compute access
allow_list = [acl[operation_field_map[operation]] for acl in acls]
allow = reduce(lambda x, y: x | y, allow_list)
if not allow:
# Deny access
raise AccessError("Access Denied -- Model: '{}' "
"Operation: '{}' - User: '******'".format(
model_name, operation, user))
return
def token(request):
""" Handles POST requests on the /api/token endpoint.
If a valid email and password are provided within the
JSON body, it responds with an access token.
"""
try:
data = request.get_json()
except BadRequest:
return JsonResponse({"msg": "Invalid JSON"}, status=400)
# Fail if data is None
if data is None:
return JsonResponse({"msg": "Invalid JSON"}, status=400)
# Make sure request body is valid
if not "email" in data or not "password" in data:
return JsonResponse({"msg": "Malformed Request"}, status=400)
conn = Connection()
user = conn.db["base.user"].find_one({"email": data["email"]})
# Check user exists
if not user:
return JsonResponse({"msg": "Bad Username or Password"}, status=401)
# Check password is valid
if not check_password_hash(user["password"], data["password"]):
return JsonResponse({"msg": "Bad Username or Password"}, status=401)
# Calculate token expiration time
token_expiration = datetime.datetime.now() + datetime.timedelta(
seconds=config.JWT_EXPIRATION_TIME)
# The token payload is composed with the email and the expiration time.
# There's no need to store it in database.
payload = {
"uid": str(user["_id"]),
"expires": token_expiration.isoformat()
}
return JsonResponse({
"access_token":
jwt.encode(payload, key=config.SECRET_KEY).decode('utf-8')
})
def check_access(docset, operation, skip_DLS=False):
"""
Check if a given user can perform
a given operation on a given model
"""
# Read docset attributes
model_name = docset._name
uid = docset.env.context["uid"]
# Get session from docset
session = docset.env.session
# Root user bypasses all security checks
if uid == ObjectId("000000000000000000000000"):
return
conn = Connection()
user = conn.db["base.user"].find_one({"_id": uid}, session=session)
if not user:
raise AccessError("User not found")
# Search in user/group many2many intermediate collection
# for groups this is user is related to.
user_group_rels = conn.db["base.user.group.rel"].find({"user_oid": uid},
session=session)
# Create a list of groups this user belongs to
groups = [rel["group_oid"] for rel in user_group_rels]
# Abort right here if user doesn't belong to any groups
if not groups:
raise AccessError("Access Denied -- Model: '{}' "
"Operation: '{}' - User: '******'".format(
model_name, operation, user["_id"]))
# The following function calls will raise an AccessError
# if user doesn't have the required privileges to operate
# the requested model.
check_ACL(model_name, operation, groups, user, session)
if not skip_DLS:
check_DLS(model_name, operation, groups, user, docset, session)
return
def start(self):
# Prevent reinitializing scheduler with this method
if self.jobs is not None:
return
# Prevent from starting if explicitly disabled in config
if config.SCHEDULER_DISABLE:
logger.warning("Unable to start scheduler (disabled per config)")
return
# Prevent from starting while in shell context
ctx = AppContext()
if ctx.read("shell"):
logger.warning("Scheduler is disabled in shell context")
return
conn = Connection()
# Get all active jobs
jobs = conn.db["base.cron"].find({"active": True})
# Update all nextcalls older than current time
for job in jobs:
now = datetime.now()
if now > job["nextcall"]:
delta_kwarg = {job["interval_type"]: job["interval"]}
new_nextcall = now + relativedelta(**delta_kwarg)
conn.db["base.cron"].update_one(
{"_id": job["_id"]}, {"$set": {
"nextcall": new_nextcall
}})
# Iterate again to get updated values
# and load them into memory
jobs.rewind()
self.jobs = dict()
for job in jobs:
self.jobs[job["_id"]] = job
self.running = True
self.process = threading.Thread(name="SchedulerLoop", target=self.loop)
self.process.start()
def check_ACL(model_name, operation, groups, user, session):
"""
Computes the ACL (Access Control List)
Raises AccessError if the given user cannot perform
the requested operation on the requested model.
"""
# Search for ACL Rules associated to all of these groups
# and also for ACL Ruless not associated to any group (Globals)
conn = Connection()
acl_rules = conn.db["base.acl"].find(
{
"active": True,
"model": model_name,
"$or": [{
"group_id": {
"$in": groups
}
}, {
"group_id": None
}]
},
session=session)
# Compute access
allow_list = [
rule[acl_operation_field_map[operation]] for rule in acl_rules
]
if allow_list:
allow = reduce(lambda x, y: x | y, allow_list)
else:
allow = []
if not allow:
# Deny access
raise AccessError("Access Denied -- Model: '{}' "
"Operation: '{}' - User: '******'".format(
model_name, operation, user["_id"]))
return
def function_wrapper(*args, **kwargs):
request = args[0]
access_token = request.headers.get("Authorization", None)
# Make sure header is present and it's valid
if not access_token or not access_token.startswith("Bearer "):
return JsonResponse(
{"msg": "Missing or Invalid Authorization Header"}, status=401)
# Attempt to decode
try:
payload = jwt.decode(access_token[7:], key=config.SECRET_KEY)
assert ({"expires", "uid"} <= set(payload.keys()))
except Exception:
return JsonResponse({"msg": "Invalid Token"}, status=401)
# Check if token is expired
fmt_str = r"%Y-%m-%dT%H:%M:%S.%f"
if datetime.datetime.now() > datetime.datetime.strptime(
payload["expires"], fmt_str):
return JsonResponse({"msg": "Access Token Has Expired"},
status=401)
# Try to create ObjectID out of str
try:
oid = ObjectId(payload["uid"])
except TypeError:
return JsonResponse({"msg": "Invalid Token"}, status=401)
# Verify if user exists in database
conn = Connection()
user = conn.db["base.user"].find_one({"_id": oid})
if not user:
# Either user was deleted or token was tampered with
return JsonResponse({"msg": "Invalid Token"}, status=401)
return func(oid, *args, **kwargs)
def build_DLS_query(model_name, operation, groups, user, session):
"""
Builds the DLS (Document Level Security) query.
This mongo query is the combination of all individual
mongo queries that affect the current user for a
given operation on a given model.
DLS rules not associated to any group are considered
as "global" and they affect everyone.
The resulting query will have the following format:
{
"$and": [
{"whatever_the_user_requested": "..."},
{"global_rule_1": "..."},
{"global_rule_2": "..."},
{
"$or": [
{"group_rule_1": "..."},
{"group_rule_2": "..."}
]
}
]
}
From the previous example, we can see global rules
constraint the domain of documents; and so do the
combination of all user specific rules. However,
one group specific rule may relax other group
specific rules previously found (but not a global one).
"""
# Root user bypasses all security checks
if user == ObjectId("000000000000000000000000"):
return False
conn = Connection()
# Search for DLS Rules associated to all of these groups
group_rules = conn.db["base.dls"].find(
{
"active": True,
dls_operation_field_map[operation]: True,
"model": model_name,
"group_id": {
"$in": groups
}
},
session=session)
# ...and also for DLS Rules not associated to any group (Globals)
global_rules = conn.db["base.dls"].find(
{
"active": True,
dls_operation_field_map[operation]: True,
"model": model_name,
"group_id": None
},
session=session)
# Initialize queries list
group_queries = []
global_queries = []
# Evaluate global expressions and add them to their list
for rule in global_rules:
query = safe_eval(rule["query"], {"user": user})
if not isinstance(query, dict):
raise ValueError(
"Invalid query in DLS rule '{}' ({}): '{}'".format(
rule.name, rule._id, rule["query"]))
global_queries.append(query)
# Evaluate user expressions and add them to their list
for rule in group_rules:
query = safe_eval(rule["query"], {"user": user})
if not isinstance(query, dict):
raise ValueError(
"Invalid query in DLS rule '{}' ({}): '{}'".format(
rule.name, rule._id, rule["query"]))
group_queries.append(query)
# Build query structure
if len(group_queries) > 0:
global_queries.append({"$or": group_queries})
if len(global_queries) > 0:
q = {"$and": global_queries}
else:
return False
return q
import logging
from olaf.fields import BaseField, Identifier, Boolean, NoPersist, RelationalField, One2many, Many2one, Many2many
from olaf.db import Connection
from bson import ObjectId
from olaf import registry
from olaf.security import check_access
logger = logging.getLogger(__name__)
conn = Connection()
class DeletionConstraintError(BaseException):
pass
class ModelMeta(type):
""" This class defines the behavior of
all model classes.
An instance of a model is always a set
of documents. Documents should have no
representation. If we'd like to work
with a single object, we have to do so
through a set of a single document.
"""
def __new__(mcs, cls, bases, dct):
for k, v in dct.items():
if isinstance(v, BaseField):
dct[k].attr = k
return super().__new__(mcs, cls, bases, dct)
def load_module_data(module_name, module_data):
"""
Routine for loading data from a given module into database
"""
def load_data(env, fname, security=False):
"""
Imports data from a file.
- If file is in CSV format, then guess the model from the filename.
- If file is in YAML format, then obtain model names from file contents.
First row represents columns, remaining rows represent the data matrix.
"""
# Get filename from abs path
base = os.path.basename(fname)
# Split basename in (name, extension)
split = os.path.splitext(base)
fields = list()
data = list()
if split[1] == ".csv":
model = split[0] if not security else "base.acl"
with open(fname) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=",")
first_line = True
for row in csv_reader:
if first_line:
first_line = False
fields = [*row]
else:
data.append([*row])
env[model].load(fields, data)
elif split[1] in [".yaml", ".yml"]:
# Parse YAML file
with open(fname) as yaml_file:
parsed_yaml = yaml.safe_load(yaml_file)
# Iterate over model names
for model_name in parsed_yaml.keys():
model = env[model_name]
# Iterate over items (document data)
for item in parsed_yaml[model_name]:
dict_data = dict()
# Iterate over field:value's
for field_name, value in item.items():
# Skip 'id' field
if field_name == "id":
dict_data[field_name] = value
continue
field = model._fields[field_name]
if isinstance(field, Many2one):
oid = env[field._comodel_name].get(value)
if not oid:
raise ValueError(
"Reference {} not found in database".
format(value))
dict_data[field_name] = value
elif isinstance(field, One2many) or isinstance(
field, Many2many):
"""
YAML Syntax Reference
model.name:
-
id: some_ext_id
some_fld: some_val
something_ids:
- create:
fld1: val1
fld2: val2
fld3: val3
- add: ref1
- add: ref2
- replace:
- ref1
- ref2
"""
tuples = list()
# TODO: Review and retest create, add and replace
# using YAML files. Create unit tests if possible.
for dd in value:
# Every item is a dictionary of one element
# {'operation_name': value}
op = next(iter(dd))
if op == "create":
value = dd[op]
elif op == "add":
value = env[field._comodel_name].get(
dd[op])
if not value:
raise ValueError(
"Reference {} not found in database"
.format(dd[op]))
elif op == "replace":
value = list()
for i in dd[op]:
oid = env[field._comodel_name].get(
i)
if not oid:
raise ValueError(
"Reference {} not found in database"
.format(i))
value.append(oid)
else:
raise ValueError(
"Operation {} not allowed during YAML data load"
.format(op))
tuples.append((op, value))
dict_data[field_name] = tuples
else:
dict_data[field_name] = value
if "id" in dict_data:
ref = env[model_name].get(dict_data["id"])
if not ref:
# Create base.model.data entry
try:
dict_data["_id"] = bson.ObjectId()
env["base.model.data"].create({
"name":
dict_data["id"],
"model":
model_name,
"res_id":
dict_data["_id"]
})
del dict_data["id"]
env[model_name].create(dict_data)
except Exception:
raise
else:
# Update found reference
del dict_data["id"]
ref.write(dict_data)
else:
# Create record with generic __import__ prefix
try:
dict_data["_id"] = bson.ObjectId()
env["base.model.data"].create({
"name":
"__import__.{}".format(dict_data["_id"]),
"model":
model_name,
"res_id":
dict_data["_id"]
})
env[model_name].create(dict_data)
except Exception:
raise
def load_file_data(env, module_name, module_data):
"""
Parses module data, gets filenames and calls loader function
"""
# Get module status
module = env.conn.db["base.module"].find_one({"name": module_name})
if not module:
result = env.conn.db["base.module"].insert_one({
"name":
module_name,
"status":
"pending"
})
module = env.conn.db["base.module"].find_one(
{"_id": result.inserted_id})
if module["status"] == "pending":
if "data" in module_data["manifest"]:
for _file in module_data["manifest"]["data"]:
fname = os.path.join(module_data["path"], module_name,
_file)
logger.debug(
"Loading data file '{}' for module '{}'".format(
_file, module_name))
load_data(env, fname)
if "security" in module_data["manifest"]:
for _file in module_data["manifest"]["security"]:
fname = os.path.join(module_data["path"], module_name,
_file)
logger.debug(
"Loading security file '{}' for module '{}'".
format(_file, module_name))
load_data(env, fname, True)
# Flag module as installed
env.conn.db["base.module"].update_one(
{"name": module_name}, {"$set": {
"status": "installed"
}})
conn = Connection()
client = conn.cl
with client.start_session() as session:
with session.start_transaction():
# Create environment with session
env = Environment(root_uid, session)
load_file_data(env, module_name, module_data)