def _get_html_template(template_file: str,
replaces: Dict[str, Any]) -> Optional[str]:
# Custom templates from project backend/models/email/
template_path = CODE_DIR.joinpath(CUSTOM_PACKAGE, MODELS_DIR, "emails",
template_file)
if not template_path.exists():
# Core templates from restapi/connectors/smtp/templates/
template_path = ABS_RESTAPI_PATH.joinpath(
CONNECTORS_FOLDER,
"smtp",
"templates",
template_file,
)
if not template_path.exists():
log.info("Template not found: {}", template_path)
return None
try:
templateLoader = jinja2.FileSystemLoader(
searchpath=template_path.parent)
templateEnv = jinja2.Environment(loader=templateLoader,
autoescape=True)
template = templateEnv.get_template(template_file)
replaces.setdefault("host", get_frontend_url())
return template.render(**replaces)
except Exception as e: # pragma: no cover
log.error("Error loading template {}: {}", template_file, e)
return None
def refresh_token(self, jti):
now = datetime.now()
token_entry = self.db.Token.query.filter_by(jti=jti).first()
if token_entry is None:
return False
if now > token_entry.expiration:
self.invalidate_token(token=token_entry.token)
log.info("This token is no longer valid: expired since {}",
token_entry.strftime("%d/%m/%Y"))
return False
# Verify IP validity only after grace period is expired
if token_entry.last_access + timedelta(
seconds=self.grace_period) < now:
ip = self.get_remote_ip()
if token_entry.IP != ip:
log.error(
"This token is emitted for IP {}, invalid use from {}",
token_entry.IP, ip)
return False
exp = now + timedelta(seconds=self.shortTTL)
token_entry.last_access = now
token_entry.expiration = exp
try:
self.db.session.add(token_entry)
self.db.session.commit()
except BaseException as e:
log.error("DB error ({}), rolling back", e)
self.db.session.rollback()
return True
def test_01_GET_status(self, client):
""" Test that the flask server is running and reachable """
# Check success
endpoint = API_URI + '/status'
alive_message = "Server is alive!"
log.info("*** VERIFY if API is online")
r = client.get(endpoint)
assert r.status_code == hcodes.HTTP_OK_BASIC
output = self.get_content(r)
assert output == alive_message
# Check failure
log.info("*** VERIFY if invalid endpoint gives Not Found")
r = client.get(API_URI)
assert r.status_code == hcodes.HTTP_BAD_NOTFOUND
# Check HTML response to status if agent/request is text/html
from restapi.rest.response import MIMETYPE_HTML
headers = {"Accept": MIMETYPE_HTML}
r = client.get(endpoint, headers=headers)
assert r.status_code == hcodes.HTTP_OK_BASIC
output = r.data.decode('utf-8')
assert output != alive_message
assert alive_message in output
assert "<html" in output
assert "<body>" in output
def init(wait):
"""Initialize data for connected services"""
if wait:
mywait()
log.info("Initialization requested")
flask_cli({'name': 'Initializing services', 'init_mode': True})
def refresh_token(self, jti):
now = datetime.now(pytz.utc)
try:
token_node = self.db.Token.nodes.get(jti=jti)
if now > token_node.expiration:
self.invalidate_token(token=token_node.token)
log.info("This token is no longer valid: expired since {}",
token_node.expiration.strftime("%d/%m/%Y"))
return False
# Verify IP validity only after grace period is expired
if token_node.last_access + timedelta(
seconds=self.grace_period) < now:
ip = self.get_remote_ip()
if token_node.IP != ip:
log.error(
"This token is emitted for IP {}, invalid use from {}",
token_node.IP, ip)
return False
exp = now + timedelta(seconds=self.shortTTL)
token_node.last_access = now
token_node.expiration = exp
token_node.save()
return True
except self.db.Token.DoesNotExist:
log.warning("Token {} not found", jti)
return False
def upload(self, subfolder: Path, force: bool = False) -> Response:
if "file" not in request.files:
raise BadRequest("No files specified")
myfile = request.files["file"]
if not myfile.filename: # pragma: no cover
raise BadRequest("Invalid filename")
if not self.allowed_file(myfile.filename):
raise BadRequest("File extension not allowed")
Uploader.validate_upload_folder(subfolder)
if not subfolder.exists():
subfolder.mkdir(parents=True, exist_ok=True)
fname = secure_filename(myfile.filename)
abs_file = subfolder.joinpath(fname)
log.info("File request for [{}]({})", myfile, abs_file)
if abs_file.exists():
if not force:
raise Conflict(
f"File '{fname}' already exists, use force parameter to overwrite"
)
abs_file.unlink()
# Save the file
try:
myfile.save(abs_file)
log.debug("Absolute file path should be '{}'", abs_file)
except Exception as e: # pragma: no cover
log.error(e)
raise ServiceUnavailable(
"Permission denied: failed to write the file")
# Check exists - but it is basicaly a test that cannot fail...
# The has just been uploaded!
if not abs_file.exists(): # pragma: no cover
raise ServiceUnavailable("Unable to retrieve the uploaded file")
########################
# ## Final response
abs_file.chmod(DEFAULT_PERMISSIONS)
# Default redirect is to 302 state, which makes client
# think that response was unauthorized....
# see http://dotnet.dzone.com/articles/getting-know-cross-origin
return EndpointResource.response(
{
"filename": fname,
"meta": self.get_file_metadata(abs_file)
},
code=200,
)
def verify_token_validity(self, jti: str, user: User) -> bool:
try:
token_entry = self.db.Token.objects.raw({"jti": jti}).first()
except self.db.Token.DoesNotExist:
return False
if token_entry.user_id is None or token_entry.user_id.email != user.email:
return False
now = datetime.now()
if now > token_entry.expiration:
self.invalidate_token(token=token_entry.token)
log.info(
"This token is no longer valid: expired since {}",
token_entry.expiration.strftime("%d/%m/%Y"),
)
return False
# Verify IP validity only after grace period is expired
if token_entry.creation + self.GRACE_PERIOD < now:
ip = self.get_remote_ip()
if token_entry.IP != ip:
log.error(
"This token is emitted for IP {}, invalid use from {}",
token_entry.IP,
ip,
)
return False
if token_entry.last_access + self.SAVE_LAST_ACCESS_EVERY < now:
token_entry.last_access = now
token_entry.save()
return True
def unpack_token(self, token, raiseErrors=False):
payload = None
try:
payload = jwt.decode(token,
self.JWT_SECRET,
algorithms=[self.JWT_ALGO])
# now > exp
except jwt.exceptions.ExpiredSignatureError as e:
# should this token be invalidated into the DB?
if raiseErrors:
raise e
else:
log.info("Unable to decode JWT token. {}", e)
# now < nbf
except jwt.exceptions.ImmatureSignatureError as e:
if raiseErrors:
raise e
else:
log.info("Unable to decode JWT token. {}", e)
except Exception as e:
if raiseErrors:
raise e
else:
log.warning("Unable to decode JWT token. {}", e)
return payload
def check_configuration(self):
try:
self.services_configuration = load_yaml_file(
file='services.yaml', path=ABS_RESTAPI_CONFSPATH)
except AttributeError as e:
log.exit(e)
for service in self.services_configuration:
name, prefix = self.prefix_name(service)
# Was this service enabled from the developer?
enable_var = str(prefix + 'enable').upper()
self.available_services[name] = self.get_bool_from_os(enable_var)
if self.available_services[name]:
# read variables
variables = self.load_variables(service, enable_var, prefix)
service['variables'] = variables
# set auth service
if name == self.authentication_name:
self.authentication_service = variables.get('service')
if self.authentication_service is None:
log.info("No service defined for authentication")
else:
log.info(
"Authentication based on '{}' service",
self.authentication_service
)
def __init__(self) -> None:
# enter GeoData in neo4j
attributes: Optional[List[str]] = None
graph = neo4j.get_instance()
with open(DATA_PATH.joinpath("geodata.tsv")) as fd:
rd = csv.reader(fd, delimiter="\t", quotechar='"')
for row in rd:
if not attributes:
# use the first row to get the list of attributes
attributes = row
else:
props = dict(zip(attributes, row))
geodata = graph.GeoData.nodes.get_or_none(
**{attributes[0]: row[0]})
if not geodata:
# create a new one
geodata = graph.GeoData(**props).save()
else:
# check if an update is needed
for key, value in props.items():
if getattr(geodata, key) != value:
setattr(geodata, key, value)
geodata.save()
log.info("GeoData nodes succesfully created")
def project_initialization(self, instances, app=None):
""" Custom initialization of your project
Please define your class Initializer in
project/YOURPROJECT/backend/initialization/initialization.py
"""
try:
# NOTE: this might be a pattern
# see in meta.py:get_customizer_class
module_path = "{}.{}.{}".format(
CUSTOM_PACKAGE,
'initialization',
'initialization',
)
module = Meta.get_module_from_string(module_path)
meta = Meta()
Initializer = meta.get_class_from_string(
'Initializer', module, skip_error=True
)
if Initializer is None:
log.debug("No custom init available")
else:
try:
Initializer(instances, app=app)
except BaseException as e:
log.error("Errors during custom initialization: {}", e)
else:
log.info("Vanilla project has been initialized")
except BaseException:
log.debug("No custom init available")
def refresh_token(self, jti):
try:
token_entry = self.db.Token.objects.raw({'jti': jti}).first()
except self.db.Token.DoesNotExist:
return False
now = datetime.now()
if now > token_entry.expiration:
self.invalidate_token(token=token_entry.token)
log.info("This token is no longer valid: expired since {}",
token_entry.strftime("%d/%m/%Y"))
return False
# Verify IP validity only after grace period is expired
if token_entry.last_access + timedelta(
seconds=self.grace_period) < now:
ip = self.get_remote_ip()
if token_entry.IP != ip:
log.error(
"This token is emitted for IP {}, invalid use from {}",
token_entry.IP, ip)
return False
exp = now + timedelta(seconds=self.shortTTL)
token_entry.last_access = now
token_entry.expiration = exp
token_entry.save()
return True
def tests(wait, core, file, folder, destroy): # pragma: no cover
"""Compute tests and coverage"""
if wait:
while initializing():
log.debug("Waiting services initialization")
time.sleep(5)
mywait()
num_opt = 0
if core:
num_opt += 1
if file is not None:
num_opt += 1
if folder is not None:
num_opt += 1
if num_opt > 1:
print_and_exit(
"Please specify only one option between --core, --file and --folder"
)
parameters = ["tests/tests.sh"]
if core:
parameters.append(current_package)
else:
parameters.append(CUSTOM_PACKAGE)
if file is not None:
# Can't be enabled due to mistral stuck at py38
# file = file.removeprefix("tests/")
if file.startswith("tests/"):
file = file[6:]
if not os.path.isfile(os.path.join("tests", file)):
print_and_exit("File not found: {}", file)
parameters.append(file)
elif folder is not None:
if not os.path.isdir(os.path.join("tests", folder)):
print_and_exit("Folder not found: {}", folder)
parameters.append(folder)
os.environ["TEST_CORE_ENABLED"] = str(core)
# In prod mode tests are execute with the server running.
# Destroy test fails with alchemy due to db locks
if destroy and not PRODUCTION:
os.environ["TEST_DESTROY_MODE"] = "1"
try:
log.info("Running tests... this may take some time")
log.debug("Executing: {}", parameters)
from plumbum import local
command = local["bash"]
command(parameters, stdout=sys.stdout, stderr=sys.stderr)
sys.exit(0)
except Exception as e:
log.error(e)
sys.exit(1)
def test_GET_status(self, client: FlaskClient) -> None:
"""Test that the flask server is running and reachable"""
# Check success
alive_message = "Server is alive"
log.info("*** VERIFY if API is online")
r = client.get(f"{API_URI}/status")
assert r.status_code == 200
output = self.get_content(r)
assert output == alive_message
# Check failure
log.info("*** VERIFY if invalid endpoint gives Not Found")
r = client.get(API_URI)
assert r.status_code == 404
if Env.get_bool("AUTH_ENABLE"):
# Check /auth/status with no token or invalid token
r = client.get(f"{AUTH_URI}/status")
assert r.status_code == 401
r = client.get(f"{AUTH_URI}/status",
headers={"Authorization": "Bearer ABC"})
assert r.status_code == 401
else:
r = client.get(f"{AUTH_URI}/status")
assert r.status_code == 404
def init_users_and_roles(self):
# Handle system roles
current_roles = []
current_roles_objs = self.db.Role.nodes.all()
for role in current_roles_objs:
current_roles.append(role.name)
log.info("Current roles: {}", current_roles)
for role in self.default_roles:
if role not in current_roles:
log.info("Creating role: {}", role)
self.create_role(role)
# Default user (if no users yet available)
if not len(self.db.User.nodes) > 0:
log.warning("No users inside graphdb. Injecting default.")
self.create_user(
{
# 'uuid': getUUID(),
'email': self.default_user,
# 'authmethod': 'credentials',
'name': 'Default',
'surname': 'User',
'password': self.default_password,
},
roles=self.default_roles,
)
else:
log.debug("Users already created")
def unpack_token(cls,
token: str,
raiseErrors: bool = False) -> Optional[DecodedPayload]:
try:
return cast(
DecodedPayload,
jwt.decode(token, cls.JWT_SECRET, algorithms=[cls.JWT_ALGO]),
)
# now > exp
except ExpiredSignatureError as e:
# should this token be invalidated into the DB?
if raiseErrors:
raise e
else:
log.info("Unable to decode JWT token. {}", e)
# now < nbf
except ImmatureSignatureError as e:
if raiseErrors:
raise e
else:
log.info("Unable to decode JWT token. {}", e)
except Exception as e:
if raiseErrors:
raise e
else:
log.warning("Unable to decode JWT token. {}", e)
return None
def send_file_streamed(
filename: str,
subfolder: Path,
mime: Optional[str] = None,
out_filename: Optional[str] = None,
) -> Response:
Uploader.validate_upload_folder(subfolder)
filename = secure_filename(filename)
filepath = subfolder.joinpath(filename)
if not filepath.is_file():
raise NotFound("The requested file does not exist")
if mime is None:
mime = Downloader.guess_mime_type(filepath)
log.info("Providing streamed content from {} (mime={})", filepath,
mime)
response = Response(
stream_with_context(Downloader.read_in_chunks(filepath)),
mimetype=mime,
)
if not out_filename:
out_filename = filepath.name
response.headers[
"Content-Disposition"] = f"attachment; filename={out_filename}"
response.headers["Content-Length"] = filepath.stat().st_size
return response
def test_04_logout(self, client: FlaskClient) -> None:
"""Check that you can logout with a valid token"""
if not Env.get_bool("AUTH_ENABLE"):
log.warning("Skipping logout tests")
return
# Check success
log.info("*** VERIFY valid token")
r = client.get(f"{AUTH_URI}/logout", headers=self.get("auth_header"))
assert r.status_code == 204
events = self.get_last_events(2)
assert events[0].event == Events.delete.value
assert events[0].user == "-"
assert events[0].target_type == "Token"
assert events[0].url == "/auth/logout"
assert events[1].event == Events.logout.value
assert events[1].user == BaseAuthentication.default_user
assert events[1].url == "/auth/logout"
# Check failure
log.info("*** VERIFY invalid token")
r = client.get(f"{AUTH_URI}/logout")
assert r.status_code == 401
def wrapper(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except BaseException:
task_id = self.request.id
task_name = self.request.task
log.error("Celery task {} failed ({})", task_id, task_name)
arguments = str(self.request.args)
log.error("Failed task arguments: {}", arguments[0:256])
log.error("Task error: {}", traceback.format_exc())
if send_mail_is_active():
log.info("Sending error report by email", task_id, task_name)
body = """
Celery task {} failed
Name: {}
Arguments: {}
Error: {}
""".format(task_id, task_name, str(self.request.args), traceback.format_exc())
project = get_project_configuration(
"project.title",
default='Unkown title',
)
subject = "{}: task {} failed".format(project, task_name)
send_mail(body, subject)
def sendmail(from_address, dest_addresses, msg):
if from_address == "invalid1":
raise SMTPException("SMTP Error")
if from_address == "invalid2":
raise BaseException("Generic Error")
fpath = "/logs/mock.mail.lastsent.json"
data = {"from": from_address, "cc": dest_addresses, "msg": msg}
log.info("Mail mock sending email from {} to {}", from_address,
dest_addresses)
with open(fpath, "w+") as file:
file.write(json.dumps(data))
log.info("Mail mock sent email from {} to {}", from_address,
dest_addresses)
log.info("Mail mock mail written in {}", fpath)
log.info("Extracting body")
fpath = "/logs/mock.mail.lastsent.body"
b = email.message_from_string(msg)
if b.is_multipart():
# get the first payload (the non html version)
first_payload = b.get_payload()[0]
payload = first_payload.get_payload()
else:
payload = b.get_payload()
with open(fpath, "w+") as file:
file.write(payload)
log.info("Mail body written in {}", fpath)
def test_destroy() -> None:
# Only executed if tests are run with --destroy flag
if os.getenv("TEST_DESTROY_MODE", "0") != "1":
log.info("Skipping destroy test, TEST_DESTROY_MODE not enabled")
return
# Always enable during core tests
if not Connector.check_availability("authentication"): # pragma: no cover
log.warning("Skipping authentication test: service not available")
return
# if Connector.check_availability("sqlalchemy"):
# sql = sqlalchemy.get_instance()
# # Close previous connections, otherwise the new create_app will hang
# sql.session.remove()
# sql.session.close_all()
auth = Connector.get_authentication_instance()
user = auth.get_user(username=BaseAuthentication.default_user)
assert user is not None
create_app(mode=ServerModes.DESTROY)
try:
auth = Connector.get_authentication_instance()
user = auth.get_user(username=BaseAuthentication.default_user)
assert user is None
except ServiceUnavailable:
pass
def unpack_token(cls, token: str, raiseErrors: bool = False) -> Optional[Payload]:
try:
if cls.JWT_SECRET:
return jwt.decode(token, cls.JWT_SECRET, algorithms=[cls.JWT_ALGO])
else:
print_and_exit( # pragma: no cover
"Server misconfiguration, missing jwt configuration"
)
# now > exp
except ExpiredSignatureError as e:
# should this token be invalidated into the DB?
if raiseErrors:
raise e
else:
log.info("Unable to decode JWT token. {}", e)
# now < nbf
except ImmatureSignatureError as e:
if raiseErrors:
raise e
else:
log.info("Unable to decode JWT token. {}", e)
except Exception as e:
if raiseErrors:
raise e
else:
log.warning("Unable to decode JWT token. {}", e)
return None
def store(self, chunk_size: int = 10000) -> None:
self.close()
log.info(
"Storing {} ({})-[:{}]->({}) relationships",
self.count,
self.label1,
self.relation,
self.label2,
)
field1 = self.fields[0].split(":")[0]
field2 = self.fields[1].split(":")[0]
properties: str = self.get_properties(self.fields[2:])
cypher = f"""
LOAD CSV WITH HEADERS FROM 'file:///{self.filename}' AS line FIELDTERMINATOR '\t'
CALL {{
WITH line
MATCH (node1: {self.label1} {{{self.key1}: line.{field1}}})
MATCH (node2: {self.label2} {{{self.key2}: line.{field2}}})
MERGE (node1)-[:{self.relation} {{{properties}}}]->(node2)
}} IN TRANSACTIONS OF {chunk_size} ROWS
"""
self.cypher_exec(cypher)
def check_authorized(self, update, context, required_admin=False):
user = update.message.from_user
user_id = user.id
text = update.message.text
if self.is_authorized(user_id, required_admin):
log.info(f"User {user_id} requested: {text}")
return True
msg = "Unauthorized request!\n"
for key, value in update.message.__dict__.items():
if key == "from_user":
for k, v in value.__dict__.items():
if not k.startswith("_") and v is not None:
msg += f"{k}: {v}\n"
if key in ["date", "photo", "text"]:
# print(key, value)
if key == "text":
msg += f"{key}: {value}\n"
log.warning(msg)
# Notify admins about violation
self.admins_broadcast(msg)
self.updater.bot.send_message(
chat_id=update.message.chat_id,
text=
"Permission denied, you are not authorized to execute this command",
)
return False
def start(self):
self.updater.start_polling(read_latency=5)
self.admins_broadcast("Bot is ready to accept requests")
log.info("Bot is ready to accept requests")
self.updater.idle()
print_and_exit("Bot closed") # pragma: no cover
def decorator(func):
log.info("Registering {}", cmd)
self.updater.dispatcher.add_handler(
CommandHandler(cmd, func, pass_args=True, run_async=run_async))
self.commands[cmd] = help
return func
def read_in_streaming(self, absolute_path, headers=None):
"""
Reads obj from iRODS without saving a local copy
"""
log.info(
"Downloading file {} in streaming with chunk size {}",
absolute_path,
self.chunk_size,
)
try:
obj = self.prc.data_objects.get(absolute_path)
# NOTE: what about binary option?
handle = obj.open('r')
if headers is None:
headers = {}
return Response(
stream_with_context(
self.read_in_chunks(handle, self.chunk_size)),
headers=headers,
)
except iexceptions.DataObjectDoesNotExist:
raise IrodsException(
"This path does not exist or permission denied")
except iexceptions.CollectionDoesNotExist:
raise IrodsException(
"This path does not exist or permission denied")
def project_init(options: Optional[Dict[str, bool]] = None) -> None:
if Connector.authentication_service != NO_AUTH:
authentication_instance = Connector.get_authentication_instance()
connector_module = Connector.get_module(
Connector.authentication_service, BACKEND_PACKAGE
)
connector = connector_module.get_instance()
log.debug("Initializing {}", Connector.authentication_service)
connector.initialize()
if options is None:
options: Dict[str, bool] = {}
with Connector.app.app_context():
authentication_instance.init_auth_db(options)
log.info("Initialized authentication module")
initializer = mem.initializer()
if initializer:
log.info("Vanilla project has been initialized")
else: # pragma: no cover
log.error("Errors during custom initialization")
if TESTING:
# Core test initialization
initialize_testing_environment(authentication_instance)
# Custom test initialization
initializer.initialize_testing_environment()
def upload_chunked(self, destination, force=False, chunk_size=None):
# Default chunk size, put this somewhere
if chunk_size is None:
chunk_size = 1048576
if os.path.exists(destination):
log.warning("Already exists")
if force:
os.remove(destination)
log.debug("Forced removal")
else:
log.error("File '{}' already exists", destination)
return False
with open(destination, "ab") as f:
while True:
chunk = request.stream.read(chunk_size)
if not chunk:
break
f.write(chunk)
# Check exists
if not os.path.exists(destination):
log.error("Unable to recover the uploaded file: {}", destination)
return False
log.info("File uploaded: {}", destination)
return True
def refresh_connection(self):
if self.db.url is None:
log.critical("Unable to refresh neo4j connection")
return False
log.info("Refreshing neo4j connection...")
self.db.set_connection(self.db.url)
return True