def custom_init(self,
pinit=False,
pdestroy=False,
abackend=None,
**kwargs):
""" Note: we ignore args here """
# recover instance with the parent method
db = super().custom_init()
if pdestroy:
# massive destruction
client = db.connection.database
from pymongo import MongoClient
client = MongoClient(self.variables.get('host'),
int(self.variables.get('port')))
system_dbs = ['admin', 'local', 'config']
for db in client.database_names():
if db not in system_dbs:
client.drop_database(db)
log.critical("Dropped db '{}'", db)
# if pinit:
# pass
return db
def remove(self, filename, subfolder=None, skip_response=False):
""" Remove the file if requested """
abs_file = self.absolute_upload_file(filename, subfolder)
# Check file existence
if not os.path.exists(abs_file):
log.critical("File '{}' not found", abs_file)
return self.force_response(
errors={"File missing": "File requested does not exists"},
code=hcodes.HTTP_BAD_NOTFOUND,
)
# Remove the real file
try:
os.remove(abs_file)
except Exception:
log.critical("Cannot remove local file {}", abs_file)
return self.force_response(
errors={"Permissions": "Failed to remove file"},
code=hcodes.HTTP_DEFAULT_SERVICE_FAIL,
)
log.warning("Removed '{}'", abs_file)
if skip_response:
return
return self.force_response("Deleted", code=hcodes.HTTP_OK_BASIC)
def import_models(name: str,
package: str,
mandatory: bool = False) -> Dict[str, Type[Any]]:
if package == BACKEND_PACKAGE:
module_name = f"{package}.connectors.{name}.models"
else:
module_name = f"{package}.models.{name}"
try:
module = Meta.get_module_from_string(module_name,
exit_on_fail=True)
except BaseException as e:
module = None
if mandatory:
log.critical(e)
if not module:
if mandatory:
print_and_exit("Cannot load {} models from {}", name,
module_name)
return {}
return Meta.get_new_classes_from_module(module)
def inline_keyboard_button(
self, update: Update,
context: CallbackContext[UD, CD, BD]) -> None: # pragma: no cover
query = update.callback_query
if not query:
log.critical(
"Debug code: query is empty in inline_keyboard_button")
return None
# Callback queries need to be answered, even if no notification to the user
# is needed. Some clients may have trouble otherwise.
# See https://core.telegram.org/bots/api#callbackquery
query.answer()
# The data cache contains the parameters wrapper of the command function. This
# wrapper will be invoked by augmenting the original list of parameters with
# the choice obtained from the inline keyboard argument
if not query.data:
log.critical(
"Debug code: query.data is empty in inline_keyboard_button")
return None
data = data_cache.pop(query.data)
query.edit_message_text(text=f"Selected option: {data['parameter']}")
# func is the parameters wrapper of the command function
func = data["func"]
# original context args are augmented with the new choice
data["context"].args.append(data["parameter"])
# Let's invoke the parameters wrapper with the new additional parameter
func(data["update"], data["context"])
def print_and_exit(message: str, *args: Optional[str],
**kwargs: Optional[str]) -> NoReturn:
# Do not import outside the function to prevent circular imports
from restapi.utilities.logs import log
log.critical(message, *args, **kwargs)
sys.exit(1)
def custom_init(self,
pinit=False,
pdestroy=False,
abackend=None,
**kwargs):
""" Note: we ignore args here """
# recover instance with the parent method
db = super().custom_init()
# do init_app on the original flask sqlalchemy extension
db.init_app(self.app)
# careful on what you do with app context on sqlalchemy
with self.app.app_context():
# check connection
from sqlalchemy import text
sql = text('SELECT 1')
db.engine.execute(sql)
if pdestroy:
# massive destruction
log.critical("Destroy current SQL data")
db.drop_all()
if pinit:
# all is fine: now create table
# because they should not exist yet
db.create_all()
return db
def custom_init(self, pinit=False, pdestroy=False, abackend=None, **kwargs):
""" Note: we ignore args here """
# recover instance with the parent method
graph = super().custom_init()
# db.init_app(self.app)
with self.app.app_context():
if pdestroy:
log.critical("Destroy current Neo4j data")
from neomodel import clear_neo4j_database
clear_neo4j_database(graph.db)
if pinit:
auto_index = self.variables.get("autoindexing", 'True') == 'True'
if auto_index:
try:
from neomodel import remove_all_labels, install_all_labels
remove_all_labels()
install_all_labels()
except BaseException as e:
log.exit(str(e))
return graph
def refresh_connection(self):
if self.db.url is None:
log.critical("Unable to refresh neo4j connection")
return False
log.info("Refreshing neo4j connection...")
self.db.set_connection(self.db.url)
return True
def destroy(self) -> None:
graph = self.get_instance()
if self.app:
with self.app.app_context():
log.critical("Destroy current Neo4j data")
clear_neo4j_database(graph.db)
def create_token(cls, payload: Payload) -> str:
""" Generate a byte token with JWT library to encrypt the payload """
if cls.JWT_SECRET:
return jwt.encode(payload, cls.JWT_SECRET, algorithm=cls.JWT_ALGO).decode(
"ascii"
)
else: # pragma: no cover
log.critical("Server misconfiguration, missing jwt configuration")
sys.exit(1)
def wrapper(*args: Any, **kwargs: Any) -> Any:
try:
return func(*args, **kwargs)
except DatabaseDuplicatedEntry as e:
# already catched and parser, raise up
raise (e)
except DoesNotExist as e:
raise (e)
except CypherSyntaxError as e:
raise (e)
except UniqueProperty as e:
t = "already exists with label"
m = re.search(
rf"Node\([0-9]+\) {t} `(.+)` and property `(.+)` = '(.+)'",
str(e))
if m:
node = m.group(1)
prop = m.group(2)
val = m.group(3)
error = f"A {node.title()} already exists with {prop}: {val}"
raise DatabaseDuplicatedEntry(error)
# Can't be tested, should never happen except in case of new neo4j version
log.error("Unrecognized error message: {}", e) # pragma: no cover
raise DatabaseDuplicatedEntry(
"Duplicated entry") # pragma: no cover
except RequiredProperty as e:
# message = property 'xyz' on objects of class XYZ
message = str(e)
m = re.search(r"property '(.*)' on objects of class (.*)", str(e))
if m:
missing_property = m.group(1)
model = m.group(2)
message = f"Missing property {missing_property} required by {model}"
raise DatabaseMissingRequiredProperty(message)
except DeflateError as e:
log.warning(e)
return None
except ServiceUnavailable: # pragma: no cover
# refresh_connection()
raise
# Catched in case of re-raise for example RequiredProperty -> BadRequest
except RestApiException: # pragma: no cover
raise
except Exception as e: # pragma: no cover
log.critical("Raised unknown exception: {}", type(e))
raise e
def manage_missing_parameter(
self,
func: Any,
param: str,
definition: fields.Field,
update: Update,
context: CallbackContext[UD, CD, BD],
error: List[str],
) -> None:
if not update.message: # pragma
log.critical(
"Debug code: missing message in manage_missing_parameter")
return None
# Parameters without description should raise some kind of errors/warnings?
if "description" in definition.metadata:
description = definition.metadata["description"]
else: # pragma: no cover
description = "???"
# Enum -> InlineKeyboardButton
if definition.validate and isinstance(definition.validate,
validate.OneOf):
choices = definition.validate.choices
labels = definition.validate.labels
if len(tuple(labels)) != len(tuple(choices)):
labels = choices
keyboard = []
for k, val in dict(zip(choices, labels)).items():
data_key = getUUID()
# Because func, update and context are not (easily) serializable they
# are saved in a data_cache and the callback will access them
# by using the assigned unique data_key
data_cache[data_key] = {
"func": func,
"update": update,
"context": context,
"parameter": k,
}
# All InlineKeyboardButton are registered with one single callback
# function (SIGH and SOB!!). The data_key passed as callback_data will
# be used to access the specific data and call again the command func
# by augmenting the parameters list with the choice from the button
keyboard.append(
[InlineKeyboardButton(val, callback_data=data_key)])
reply_markup = InlineKeyboardMarkup(keyboard)
update.message.reply_text(description, reply_markup=reply_markup)
# Other errors
# Never raised during tests
else: # pragma: no cover
update.message.reply_text(f"{description}\n{param}: {error[0]}")
def my_self(update: Update, context: CallbackContext[UD, CD, BD]) -> None:
# Can't be true, since it is restricted_to_admins
if not update.message or not update.message.from_user: # pragma: no cover
log.critical("Error: user is missing")
return None
user_firstname = update.message.from_user.first_name
user_id = update.message.from_user.id
update.message.reply_text(
f"Hello {user_firstname}, your Telegram ID is {user_id}")
def get_authentication_instance() -> BaseAuthentication:
if not Connector._authentication_module:
Connector._authentication_module = Connector.get_module(
Connector.authentication_service, BACKEND_PACKAGE
)
if not Connector._authentication_module: # pragma: no cover
log.critical("{} not available", Connector.authentication_service)
raise ServiceUnavailable("Authentication service not available")
return Connector._authentication_module.Authentication()
def test_caching_autocleaning(self, client: FlaskClient) -> None:
if Env.get_bool("AUTH_ENABLE"):
headers, _ = self.do_login(client, None, None)
else:
headers = None
# Syncronize this test to start at the beginning of the next second and
# prevent the test to overlap a change of second
# Since the caching is rounded to the second, few milliseconds cann make the
# difference, for example:
# A first request at 00:00:00.997 is cached
# A second request at 00:00:01.002 is no longer cached, even if only 5 millisec
# elapsed because the second changed
# Added 0.01 just to avoid to exactly start at the beginning of the second
t = 1.01 - datetime.now().microsecond / 1000000.0
log.critical("Sleeping {} sec", t)
time.sleep(t)
# the GET method is cached for 1 second
# First response is not cached
r = client.get(f"{API_URI}/tests/cache/short")
assert r.status_code == 200
counter1 = self.get_content(r)
# Second response is cached
r = client.get(f"{API_URI}/tests/cache/short")
assert r.status_code == 200
assert self.get_content(r) == counter1
# Third response is no longer cached
time.sleep(1)
r = client.get(f"{API_URI}/tests/cache/short")
assert r.status_code == 200
counter2 = self.get_content(r)
assert counter2 != counter1
# Fourth response is cached again
r = client.get(f"{API_URI}/tests/cache/short")
assert r.status_code == 200
assert self.get_content(r) == counter2
# Endpoint is unauthenticated, headers are ignored when building the cache key
r = client.get(f"{API_URI}/tests/cache/short", headers=headers)
assert r.status_code == 200
assert self.get_content(r) == counter2
# Tokens are ignored even if invalid
r = client.get(f"{API_URI}/tests/cache/short",
headers={"Authorization": "Bearer invalid"})
assert r.status_code == 200
assert self.get_content(r) == counter2
def wrapper(self, *args, **kwargs):
out = None
try:
out = func(self, *args, **kwargs)
# Catch the exception requested by the user
except RestApiException as e:
if e.is_warning:
log.warning(e)
else:
log.exception(e)
log.error(e)
return self.response(e.args[0], code=e.status_code)
except werkzeug.exceptions.BadRequest: # pragma: no cover
# do not stop werkzeug BadRequest
raise
except werkzeug.exceptions.UnprocessableEntity:
# do not stop werkzeug UnprocessableEntity, it will be
# catched by handle_marshmallow_errors
raise
# raised in case of malformed Range header
except werkzeug.exceptions.RequestedRangeNotSatisfiable:
raise
# Catch any other exception
# errors with RabbitMQ credentials raised when sending Celery tasks
except AccessRefused as e: # pragma: no cover
log.critical(e)
return self.response("Unexpected Server Error", code=500)
except Exception as e:
if SENTRY_URL is not None: # pragma: no cover
capture_exception(e)
excname = e.__class__.__name__
message = str(e)
if not message: # pragma: no cover
message = "Unknown error"
log.exception(message)
log.error("Catched {} exception: {}", excname, message)
if excname in SYSTEM_EXCEPTIONS:
return self.response(
"Server failure; please contact admin.", code=400
)
return self.response({excname: message}, code=400)
return out
def get(self, service):
log.critical(detector.available_services)
if not detector.check_availability(service):
raise RestApiException(
"Unknown service: {}".format(service),
status_code=hcodes.HTTP_BAD_UNAUTHORIZED,
)
service_instance = self.get_service_instance(service, global_instance=False)
log.critical(service_instance)
return self.force_response("Service is reachable: {}".format(service))
def get_bindings(self, exchange: str) -> Optional[List[Dict[str, str]]]:
if not self.exchange_exists(exchange):
log.critical("Does not exist")
return None
host = self.variables.get("host", "")
schema = ""
if not host.startswith("http"):
if Env.to_bool(self.variables.get("ssl_enabled")):
schema = "https://"
else:
schema = "http://"
port = self.variables.get("management_port")
# url-encode unsafe characters by also including / (thanks to safe parameter)
# / -> %2F
vhost = urllib.parse.quote(self.variables.get("vhost", "/"), safe="")
user = self.variables.get("user")
password = self.variables.get("password")
# API Reference:
# A list of all bindings in which a given exchange is the source.
r = requests.get(
f"{schema}{host}:{port}/api/exchanges/{vhost}/{exchange}/bindings/source",
auth=HTTPBasicAuth(user, password),
verify=False,
)
response = r.json()
if r.status_code != 200: # pragma: no cover
raise RestApiException(
{"RabbitMQ": response.get("error", "Unknown error")},
status_code=r.status_code,
)
bindings = []
for row in response:
# row == {
# 'source': exchange-name,
# 'vhost': probably '/',
# 'destination': queue-or-dest-exchange-name,
# 'destination_type': 'queue' or 'exchange',
# 'routing_key': routing_key,
# 'arguments': Dict,
# 'properties_key': ?? as routing_key?
# }
bindings.append({
"exchange": row["source"],
"routing_key": row["routing_key"],
"queue": row["destination"],
})
return bindings
def send_markdown(self, msg: str, update: Update) -> None:
if not msg.strip(): # pragma: no cover
return
if update.message:
self.updater.bot.send_message(
chat_id=update.message.chat_id,
text=msg.replace("_", "-"),
parse_mode=ParseMode.MARKDOWN,
)
else: # pragma: no cover
log.critical(
"Debug code: update.message in missing in send_markdown")
def get_class_from_string(classname, module, skip_error=False):
""" Get a specific class from a module using a string variable """
myclass = None
try:
# Meta language for dinamically import
myclass = getattr(module, classname)
except AttributeError as e:
if not skip_error:
log.critical("Failed to load class from module: " + str(e))
else:
pass
return myclass
def destroy(self) -> None:
instance = self.get_instance()
if self.app:
with self.app.app_context():
sql = text("SELECT 1")
instance.db.engine.execute(sql)
instance.db.session.remove()
close_all_sessions()
# massive destruction
log.critical("Destroy current SQL data")
instance.db.drop_all()
def destroy(self) -> None:
instance = self.get_instance()
# massive destruction
client = instance.connection.database
from pymongo import MongoClient
uri = self._get_uri(self.variables)
client = MongoClient(uri)
system_dbs = ["admin", "local", "config"]
for db in client.list_database_names():
if db not in system_dbs:
client.drop_database(db)
log.critical("Dropped db '{}'", db)
def wrapper(*args, **kwargs):
# Recover the auth object
auth_type, token = HTTPTokenAuth.get_authorization_token(
allow_access_token_parameter=allow_access_token_parameter)
# Internal API 'self' reference
caller = Meta.get_self_reference_from_args(*args)
if caller is None: # pragma: no cover
# An exit here is really really dangerous, but even if
# get_self_reference_from_args can return None, this case is quite
# impossible... however with None the server can't continue!
log.critical(
"Server misconfiguration, self reference can't be None!"
)
# with print_and_exit my-py does not understand that the execute
# halts here... let's use an explicit exit
sys.exit(1)
if (auth_type is not None and auth_type == HTTPAUTH_SCHEME
and request.method != "OPTIONS"):
caller.unpacked_token = caller.auth.verify_token(token)
# Check authentication. Optional authentication is valid if:
# 1) token is missing
# 2) token is valid
# Invalid tokens are rejected
if not caller.unpacked_token[0]:
# Clear TCP receive buffer of any pending data
_ = request.data
# Mimic the response from a normal endpoint
# To use the same standards
# log.info("Invalid token received '{}'", token)
log.debug("Invalid token received")
return caller.response(
"Invalid token received",
headers=HTTPAUTH_ERR_HEADER,
code=401,
allow_html=True,
)
request.environ[TOKEN_VALIDATED_KEY] = True
return func(*args, **kwargs)
def make_login(self, username, password):
""" The method which will check if credentials are good to go """
try:
user = self.get_user_object(username=username)
except BaseException as e:
log.error("Unable to connect to auth backend\n[{}] {}", type(e), e)
# log.critical("Please reinitialize backend tables")
from restapi.exceptions import RestApiException
raise RestApiException(
"Unable to connect to auth backend",
status_code=hcodes.HTTP_SERVER_ERROR,
)
if user is None:
return None, None
try:
# Check if Oauth2 is enabled
if user.authmethod != 'credentials':
return None, None
except BaseException:
# Missing authmethod as requested for authentication
log.critical("Current authentication db models are broken!")
return None, None
# New hashing algorithm, based on bcrypt
if self.verify_password(password, user.password):
return self.create_token(self.fill_payload(user))
# old hashing; since 0.7.2. Removed me in a near future!!
if self.check_old_password(user.password, password):
log.warning(
"Old password encoding for user {}, automatic convertion",
user.email)
now = datetime.now(pytz.utc)
user.password = BaseAuthentication.get_password_hash(password)
user.last_password_change = now
self.save_user(user)
return self.make_login(username, password)
return None, None
def wrapper(self, *args, **kwargs):
from neomodel.exceptions import RequiredProperty
try:
return func(self, *args, **kwargs)
except DatabaseDuplicatedEntry as e:
log.critical("boh")
raise Conflict(str(e))
except RequiredProperty as e:
log.critical("Missing required")
raise BadRequest(e)
def invalid_message(self, update: Update,
context: CallbackContext[UD, CD, BD]) -> None:
if update.message:
user = update.message.from_user.id if update.message.from_user else "N/A"
log.info(
"Received invalid message from {}: {}",
user,
update.message.text,
)
if self.check_authorized(update, context):
self.updater.bot.send_message(
chat_id=update.message.chat_id,
text="Invalid command, ask for /help",
)
else: # pragma: no cover
log.critical(
"Debug code: update.message in missing in invalid_message")
def verify_roles(
self,
user: User,
roles: Optional[List[Union[str, Role]]],
required_roles: str = ALL_ROLES,
warnings: bool = True,
) -> bool:
if not roles:
return True
current_roles = self.get_roles_from_user(user)
if required_roles == ALL_ROLES:
for role in roles:
if isinstance(role, Role):
role = role.value
if role not in current_roles:
if warnings:
log.warning("Auth role '{}' missing for request", role)
return False
return True
if required_roles == ANY_ROLE:
for role in roles:
if isinstance(role, Role):
role = role.value
if role in current_roles:
return True
log.warning(
"Expected at least one roles from {}, found none in {}",
roles,
current_roles,
)
return False
log.critical("Unknown role authorization requirement: {}",
required_roles)
return False
def change_password(
self,
user: User,
password: str,
new_password: Optional[str],
password_confirm: Optional[str],
) -> bool:
if new_password is None:
raise BadRequest("Missing new password")
if password_confirm is None:
raise BadRequest("Missing password confirmation")
if new_password != password_confirm:
raise Conflict("Your password doesn't match the confirmation")
check, msg = self.verify_password_strength(
pwd=new_password,
old_pwd=password,
email=user.email,
name=user.name,
surname=user.surname,
)
if not check:
raise Conflict(msg)
user.password = BaseAuthentication.get_password_hash(new_password)
user.last_password_change = datetime.now(pytz.utc)
self.save_user(user)
self.log_event(Events.change_password, user=user)
for token in self.get_tokens(user=user):
try:
self.invalidate_token(token=token["token"])
except Exception as e: # pragma: no cover
log.critical("Failed to invalidate token {}", e)
return True
def init_groups(self, force):
create = False
update = False
default_group = self.get_group(name=DEFAULT_GROUP_NAME)
# If there are no groups, let's create the default group
if not self.get_groups():
create = True
# If there are some groups skip group creation in absence of a force flag
elif force:
# If force flag is enable, create the default group if missing or update it
create = default_group is None
update = default_group is not None
if create:
default_group = self.create_group(
{
"shortname": DEFAULT_GROUP_NAME,
"fullname": DEFAULT_GROUP_DESCR,
}
)
log.info("Injected default group")
elif update:
log.info("Default group already exists, updating")
# Added to make the life easier to mypy... but cannot be False
if default_group:
default_group.shortname = DEFAULT_GROUP_NAME
default_group.fullname = DEFAULT_GROUP_DESCR
else: # pragma: no cover
log.critical("Default group not found")
self.save_group(default_group)
elif default_group:
log.info("Default group already exists")
else:
log.info("Default group does not exist but other groups do")
return default_group
def check_authorized(
self,
update: Update,
context: CallbackContext[UD, CD, BD],
required_admin: bool = False,
) -> bool:
if not update.message or not update.message.from_user:
log.critical("Debug code: missing user in check_authorized")
return False
user = update.message.from_user
user_id = user.id
text = update.message.text
if self.is_authorized(user_id, required_admin):
log.info(f"User {user_id} requested: {text}")
return True
msg = "Unauthorized request!\n"
for key, value in update.message.__dict__.items():
if key == "from_user":
for k, v in value.__dict__.items():
if not k.startswith("_") and v is not None:
msg += f"{k}: {v}\n"
if key in ["date", "photo", "text"]:
# print(key, value)
if key == "text":
msg += f"{key}: {value}\n"
log.warning(msg)
# Notify admins about violation
self.admins_broadcast(msg)
self.updater.bot.send_message(
chat_id=update.message.chat_id,
text=
"Permission denied, you are not authorized to execute this command",
)
return False