def get_instance(module_relpath: str, class_name: str,
**kwargs: Any) -> Any:
MyClass = Meta.get_class(module_relpath, class_name)
if MyClass is None:
return None
try:
return MyClass(**kwargs)
except BaseException as e: # pragma: no cover
log.error("Errors loading {}.{}: {}", module_relpath, class_name,
e)
return None
def publish_on_socket(self, channel, message, sync=False):
item = Item(WebSocketMessageFormat(message, binary=False))
if not sync:
self.pub.publish(channel, item, callback=self.callback)
return True
try:
self.pub.publish(channel, item, blocking=True)
log.debug('Message successfully published on pushpin')
return True
except BaseException as e:
log.error('Publish failed on pushpin: {}', message)
log.error(e)
return False
def load_classes(self):
for service in self.services_configuration:
name, _ = self.prefix_name(service)
if not self.available_services.get(name):
continue
log.verbose("Looking for class {}", name)
variables = service.get('variables')
ext_name = service.get('class')
# Get the existing class
try:
MyClass = self.load_class_from_module(ext_name, service=service)
# Passing variables
MyClass.set_variables(variables)
if service.get('load_models'):
base_models = self.meta.import_models(
name, BACKEND_PACKAGE, exit_on_fail=True
)
if EXTENDED_PACKAGE == EXTENDED_PROJECT_DISABLED:
extended_models = {}
else:
extended_models = self.meta.import_models(
name, EXTENDED_PACKAGE, exit_on_fail=False
)
custom_models = self.meta.import_models(
name, CUSTOM_PACKAGE, exit_on_fail=False
)
MyClass.set_models(base_models, extended_models, custom_models)
except AttributeError as e:
log.error(str(e))
log.exit('Invalid Extension class: {}', ext_name)
# Save
self.services_classes[name] = MyClass
log.debug("Got class definition for {}", MyClass)
if len(self.services_classes) < 1:
raise KeyError("No classes were recovered!")
return self.services_classes
def error_callback(self, update, context):
# https://github.com/python-telegram-bot/python-telegram-bot/wiki/Exception-Handling
if isinstance(context.error, TooManyInputs):
update.message.reply_text("Too many inputs",
parse_mode=ParseMode.MARKDOWN)
# Two instances running on the same account
elif isinstance(context.error, TelegramConflict): # pragma: no cover
self.admins_broadcast(str(context.error))
log.warning("Stopping bot...")
self.shutdown()
# Never happens during tests... how to test it?
else: # pragma: no cover
log.error(context.error)
self.admins_broadcast(str(context.error))
def get(self):
current_user = self.get_current_user()
data = {
'uuid': current_user.uuid,
'status': "Valid user",
'email': current_user.email,
}
# roles = []
roles = {}
for role in current_user.roles:
# roles.append(role.name)
roles[role.name] = role.description
data["roles"] = roles
try:
for g in current_user.belongs_to.all():
data["group"] = {
"uuid": g.uuid,
"shortname": g.shortname,
"fullname": g.fullname,
}
except BaseException as e:
log.verbose(e)
data["isAdmin"] = self.auth.verify_admin()
data["isLocalAdmin"] = self.auth.verify_local_admin()
if hasattr(current_user, 'privacy_accepted'):
data["privacy_accepted"] = current_user.privacy_accepted
if hasattr(current_user, 'name'):
data["name"] = current_user.name
if hasattr(current_user, 'surname'):
data["surname"] = current_user.surname
if self.auth.SECOND_FACTOR_AUTHENTICATION is not None:
data['2fa'] = self.auth.SECOND_FACTOR_AUTHENTICATION
obj = meta.get_customizer_class('apis.profile', 'CustomProfile')
if obj is not None:
try:
data = obj.manipulate(ref=self, user=current_user, data=data)
except BaseException as e:
log.error("Could not custom manipulate profile:\n{}", e)
return self.force_response(data)
def get_schema_type(field: str,
schema: marshmallow_fields.Field,
default: Optional[Any] = None) -> str:
if schema.metadata.get("password", False):
return "password"
# types from https://github.com/danohu/py2ng
# https://github.com/danohu/py2ng/blob/master/py2ng/__init__.py
if isinstance(schema, fields.Bool) or isinstance(
schema, fields.Boolean):
return "boolean"
if isinstance(schema, fields.Date):
return "date"
# Include both AwareDateTime and NaiveDateTime that extend DateTime
if isinstance(schema, fields.DateTime):
return "datetime"
if isinstance(schema, fields.Decimal):
return "number"
if isinstance(schema, fields.Dict):
return "dictionary"
if isinstance(schema, fields.Email):
return "email"
# if isinstance(schema, fields.Field):
# return 'any'
if isinstance(schema, fields.Float):
return "number"
if isinstance(schema, fields.Int) or isinstance(
schema, fields.Integer):
return "int"
if isinstance(schema, fields.List):
key = schema.data_key or field
inner_type = ResponseMaker.get_schema_type(field,
schema.inner,
default=key)
return f"{inner_type}[]"
if isinstance(schema, fields.Nested):
return "nested"
if isinstance(schema, fields.Number):
return "number"
if isinstance(schema, fields.Str) or isinstance(schema, fields.String):
return "string"
# Reached with lists of custom types
if default:
return str(default)
log.error("Unknown schema type: {}", type(schema))
return "string"
def publish_on_stream(self, channel, message, sync=False):
if not sync:
self.pub.publish_http_stream(channel,
message,
callback=self.callback)
return True
try:
self.pub.publish_http_stream(channel, message, blocking=True)
log.debug('Message successfully published on pushpin')
return True
except BaseException as e:
log.error('Publish failed on pushpin: {}', message)
log.error(e)
return False
def invalidate_all_tokens(self, user=None):
"""
To invalidate all tokens the user uuid is changed
"""
if user is None:
user = self._user
user.uuid = getUUID()
try:
self.db.session.add(user)
self.db.session.commit()
log.warning("User uuid changed to: {}", user.uuid)
except BaseException as e:
log.error("DB error ({}), rolling back", e)
self.db.session.rollback()
return True
def custom_init(self, pinit=False, pdestroy=False, abackend=None, **kwargs):
# Get the instance from the parent
obj = super().custom_init()
# NOTE: Inject the backend as the object 'db' inside the instance
# IMPORTANT!!! this is the 'hat trick' that makes things possible
obj.db = abackend
if pinit:
with self.app.app_context():
obj.init_users_and_roles()
log.info("Initialized authentication module")
if pdestroy:
log.error("Destroy not implemented for authentication service")
def _deserialize(
self,
value: Any,
attr: Optional[str],
data: Optional[Mapping[str, Any]],
**kwargs: Any,
) -> Any:
value = super()._deserialize(value, attr, data, **kwargs)
if not re.match(r"^[0-9]{6}$", value):
if TESTING:
log.error("Invalid TOTP format: {}", value)
raise ValidationError("Invalid TOTP format")
return value
def obfuscate_query_parameters(raw_url: str) -> str:
url = urllib_parse.urlparse(raw_url)
try:
params = urllib_parse.unquote(
urllib_parse.urlencode(handle_log_output(url.query)))
url = url._replace(query=params)
# remove http(s)://
url = url._replace(scheme="")
# remove hostname:port
url = url._replace(netloc="")
except TypeError: # pragma: no cover
log.error("Unable to url encode the following parameters:")
print(url.query)
return urllib_parse.urlunparse(url)
return urllib_parse.urlunparse(url)
def update_profile(self, user, data):
avoid_update = ['uuid', 'authmethod', 'is_active', 'roles']
try:
for key, value in data.items():
if key.startswith('_') or key in avoid_update:
continue
log.debug("Profile new value: {}={}", key, value)
setattr(user, key, value)
except BaseException as e:
log.error("Failed to update profile:\n{}: {}", e.__class__.__name__, e)
else:
log.info("Profile updated")
self.auth.save_user(user)
def custom_post_handle_user_input(self, user_node, input_data):
module_path = "{}.initialization.initialization".format(CUSTOM_PACKAGE)
module = Meta.get_module_from_string(module_path)
meta = Meta()
Customizer = meta.get_class_from_string('Customizer',
module,
skip_error=True)
if Customizer is None:
log.debug("No user properties customizer available")
else:
try:
Customizer().custom_post_handle_user_input(
self, user_node, input_data)
except BaseException as e:
log.error("Unable to customize user properties: {}", e)
def validate_upload_folder(path: Path) -> None:
if "\x00" in str(path):
raise BadRequest("Invalid null byte in subfolder parameter")
if path != path.resolve():
log.error("Invalid path: path is relative or contains double-dots")
raise Forbidden("Invalid file path")
if path != DATA_PATH and DATA_PATH not in path.parents:
log.error(
"Invalid root path: {} is expected to be a child of {}",
path,
DATA_PATH,
)
raise Forbidden("Invalid file path")
def verify_token(
self,
token: Optional[str],
raiseErrors: bool = False,
token_type: Optional[str] = None,
) -> Tuple[bool, Optional[str], Optional[str], Optional[User]]:
if token is None:
if raiseErrors:
raise InvalidToken("Missing token")
return self.unpacked_token(False)
# Decode the current token
payload = self.unpack_token(token, raiseErrors=raiseErrors)
if payload is None:
if raiseErrors:
raise InvalidToken("Invalid payload")
return self.unpacked_token(False)
payload_type = payload.get("t", self.FULL_TOKEN)
if token_type is None:
token_type = self.FULL_TOKEN
if token_type != payload_type:
log.error("Invalid token type {}, required: {}", payload_type, token_type)
if raiseErrors:
raise InvalidToken("Invalid token type")
return self.unpacked_token(False)
# Get the user from payload
user = self.get_user(user_id=payload.get("user_id"))
if user is None:
if raiseErrors:
raise InvalidToken("No user from payload")
return self.unpacked_token(False)
# implemented from the specific db services
if not self.verify_token_validity(jti=payload["jti"], user=user):
if raiseErrors:
raise InvalidToken("Token is not valid")
return self.unpacked_token(False)
log.debug("User {} authorized", user.email)
return self.unpacked_token(True, token=token, jti=payload["jti"], user=user)
def invalidate_token(self, token: str) -> bool:
token_entry = self.db.Token.query.filter_by(token=token).first()
if token_entry:
try:
# Call to untyped function "delete" in typed context
self.db.session.delete(token_entry) # type: ignore
self.db.session.commit()
self.log_event(Events.delete, target=token_entry)
return True
except Exception as e: # pragma: no cover
log.error("Could not invalidate token ({}), rolling back", e)
self.db.session.rollback()
return False
log.warning("Could not invalidate token")
return False
def make_login(self, username: str, password: str) -> Tuple[str, Payload, User]:
""" The method which will check if credentials are good to go """
try:
user = self.get_user(username=username)
except ValueError as e: # pragma: no cover
# SqlAlchemy can raise the following error:
# A string literal cannot contain NUL (0x00) characters.
log.error(e)
raise BadRequest("Invalid input received")
except BaseException as e: # pragma: no cover
log.error("Unable to connect to auth backend\n[{}] {}", type(e), e)
raise ServiceUnavailable("Unable to connect to auth backend")
if user is None:
self.register_failed_login(username)
self.log_event(
Events.failed_login,
payload={"username": username},
user=user,
)
raise Unauthorized("Invalid access credentials", is_warning=True)
# Check if Oauth2 is enabled
if user.authmethod != "credentials": # pragma: no cover
raise BadRequest("Invalid authentication method")
# New hashing algorithm, based on bcrypt
if self.verify_password(password, user.password):
# Token expiration is capped by the user expiration date, if set
payload, full_payload = self.fill_payload(user, expiration=user.expiration)
token = self.create_token(payload)
self.log_event(Events.login, user=user)
return token, full_payload, user
self.log_event(
Events.failed_login,
payload={"username": username},
user=user,
)
self.register_failed_login(username)
raise Unauthorized("Invalid access credentials", is_warning=True)
def make_login(self, username, password):
""" The method which will check if credentials are good to go """
try:
user = self.get_user_object(username=username)
except BaseException as e:
log.error("Unable to connect to auth backend\n[{}] {}", type(e), e)
# log.critical("Please reinitialize backend tables")
from restapi.exceptions import RestApiException
raise RestApiException(
"Unable to connect to auth backend",
status_code=hcodes.HTTP_SERVER_ERROR,
)
if user is None:
return None, None
try:
# Check if Oauth2 is enabled
if user.authmethod != 'credentials':
return None, None
except BaseException:
# Missing authmethod as requested for authentication
log.critical("Current authentication db models are broken!")
return None, None
# New hashing algorithm, based on bcrypt
if self.verify_password(password, user.password):
return self.create_token(self.fill_payload(user))
# old hashing; since 0.7.2. Removed me in a near future!!
if self.check_old_password(user.password, password):
log.warning(
"Old password encoding for user {}, automatic convertion",
user.email)
now = datetime.now(pytz.utc)
user.password = BaseAuthentication.get_password_hash(password)
user.last_password_change = now
self.save_user(user)
return self.make_login(username, password)
return None, None
def tests(wait, core, file, folder):
"""Compute tests and coverage"""
if wait:
while starting_up():
log.debug('Waiting service startup')
time.sleep(5)
mywait()
log.debug("Starting unit tests: {}", pretty_errors)
# launch unittests and also compute coverage
log.warning("Running all tests and computing coverage.\n" +
"This may take some minutes.")
parameters = []
if core:
parameters.append(current_package)
elif file is not None:
if not os.path.isfile(os.path.join("tests", file)):
log.exit("File not found: {}", file)
else:
parameters.append("default")
parameters.append(file)
elif folder is not None:
if not os.path.isdir(os.path.join("tests", folder)):
log.exit("Folder not found: {}", folder)
else:
parameters.append("default")
parameters.append(folder)
try:
# TODO: convert the `pyunittests` script from the docker image into python
# Pattern in plumbum library for executing a shell command
from plumbum import local
command = local["pyunittests"]
log.verbose("Executing command pyunittests {}", parameters)
output = command(parameters)
except Exception as e:
log.error(str(e))
raise e
log.info("Completed:\n{}", output)
def create_user(self, user, admin=False):
if user is None:
log.error("Asking for NULL user...")
return False
user_type = 'rodsuser'
if admin:
user_type = 'rodsadmin'
try:
user_data = self.prc.users.create(user, user_type)
log.info("Created user: {}", user_data)
except iexceptions.CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME:
log.warning("User {} already exists in iRODS", user)
return False
return True
def get_html_template(template_file, replaces):
try:
template_path = os.path.join(os.curdir, CUSTOM_PACKAGE, MODELS_DIR,
"emails")
templateLoader = jinja2.FileSystemLoader(searchpath=template_path)
templateEnv = jinja2.Environment(loader=templateLoader,
autoescape=True)
template = templateEnv.get_template(template_file)
return template.render(**replaces)
except jinja2.exceptions.TemplateNotFound as e:
log.error("Template not found: {} ({})", template_file, e)
return None
except BaseException as e:
log.error("Error loading template {}: {}", template_file, e)
return None
def validation(self, swag_dict):
"""
Based on YELP library,
verify the current definition on the open standard
"""
if len(swag_dict['paths']) < 1:
raise AttributeError("Swagger 'paths' definition is empty")
filepath = os.path.join(tempfile.gettempdir(), 'test.json')
try:
# Fix jsonschema validation problem
# expected string or bytes-like object
# http://j.mp/2hEquZy
swag_dict = json.loads(json.dumps(swag_dict))
# write it down
# FIXME: can we do better than this?
with open(filepath, 'w') as f:
json.dump(swag_dict, f)
except Exception as e:
raise e
# log.warning("Failed to temporary save the swagger definition")
bravado_config = {
'validate_swagger_spec': True,
'validate_requests': False,
'validate_responses': False,
'use_models': False,
}
try:
self._customizer._validated_spec = Spec.from_dict(
swag_dict, config=bravado_config)
log.debug("Swagger configuration is validated")
except Exception as e:
# raise e
error = str(e).split('\n')[0]
log.error("Failed to validate:\n{}\n", error)
return False
finally:
os.remove(filepath)
return True
def copy(
self,
sourcepath,
destpath,
recursive=False,
force=False,
compute_checksum=False,
compute_and_verify_checksum=False,
):
if recursive:
log.error("Recursive flag not implemented for copy")
if self.is_collection(sourcepath):
raise IrodsException("Copy directory not supported")
if compute_checksum:
raise IrodsException("Compute_checksum not supported in copy")
if compute_and_verify_checksum:
raise IrodsException(
"Compute_and_verify_checksum not supported in copy")
if sourcepath == destpath:
raise IrodsException("Source and destination path are the same")
try:
log.verbose("Copy {} into {}", sourcepath, destpath)
source = self.prc.data_objects.get(sourcepath)
self.create_empty(destpath, directory=False, ignore_existing=force)
target = self.prc.data_objects.get(destpath)
with source.open('r+') as f:
with target.open('w') as t:
for line in f:
# if t.writable():
t.write(line)
except iexceptions.DataObjectDoesNotExist:
raise IrodsException(
"DataObject not found (or no permission): {}".format(
sourcepath))
except iexceptions.CollectionDoesNotExist:
raise IrodsException(
"Collection not found (or no permission): {}".format(
sourcepath))
def invalidate_token(self, token, user=None):
# if user is None:
# user = self.get_user()
token_entry = self.db.Token.query.filter_by(token=token).first()
if token_entry is not None:
# Token are now deleted and no longer kept with no emision info
# token_entry.emitted_for = None
try:
self.db.session.delete(token_entry)
self.db.session.commit()
return True
except BaseException as e:
log.error("Could not invalidate token ({}), rolling back", e)
self.db.session.rollback()
return False
log.warning("Could not invalidate token")
return False
def custom_extra_registration(variables):
# Add the possibility to user a custom registration extra service
oscr = detector.get_global_var('CUSTOM_REGISTER', default='noname')
obj = meta.get_customizer_class(
'apis.profile', 'CustomRegister', {'client_name': oscr}
)
if obj is not None:
try:
obj.new_member(
email=variables['email'],
name=variables['name'],
surname=variables['surname'],
)
except BaseException as e:
log.error(
"Could not register your custom profile:\n{}: {}",
e.__class__.__name__,
e,
)
def get_periodic_task(cls, name):
if cls.CELERY_BEAT_SCHEDULER == 'MONGODB':
from celerybeatmongo.models import PeriodicTask, DoesNotExist
try:
return PeriodicTask.objects.get(name=name)
except DoesNotExist:
return None
elif cls.CELERY_BEAT_SCHEDULER == 'REDIS':
from redbeat.schedulers import RedBeatSchedulerEntry
try:
task_key = "{}{}".format(cls.REDBEAT_KEY_PREFIX, name)
return RedBeatSchedulerEntry.from_key(
task_key, app=CeleryExt.celery_app)
except KeyError:
return None
else:
log.error(
"Unsupported celery-beat scheduler: {}", cls.CELERY_BEAT_SCHEDULER)
def verify_token_validity(self, jti: str, user: User) -> bool:
token_entry = self.db.Token.query.filter_by(jti=jti).first()
if token_entry is None:
return False
if token_entry.user_id is None or token_entry.user_id != user.id:
return False
# offset-naive datetime to compare with MySQL
now = get_now(token_entry.expiration.tzinfo)
if now > token_entry.expiration:
self.invalidate_token(token=token_entry.token)
log.info(
"This token is no longer valid: expired since {}",
token_entry.expiration.strftime("%d/%m/%Y"),
)
return False
# Verify IP validity only after grace period is expired
if token_entry.creation + self.GRACE_PERIOD < now:
ip = self.get_remote_ip()
if token_entry.IP != ip:
log.warning(
"This token is emitted for IP {}, invalid use from {}",
token_entry.IP,
ip,
)
return False
if token_entry.last_access + self.SAVE_LAST_ACCESS_EVERY < now:
token_entry.last_access = now
try:
self.db.session.add(token_entry)
self.db.session.commit()
except Exception as e: # pragma: no cover
log.error("DB error ({}), rolling back", e)
self.db.session.rollback()
return True
def custom_user_properties(self, userdata):
module_path = "{}.initialization.initialization".format(CUSTOM_PACKAGE)
module = Meta.get_module_from_string(module_path)
meta = Meta()
Customizer = meta.get_class_from_string('Customizer',
module,
skip_error=True)
if Customizer is None:
log.debug("No user properties customizer available")
else:
try:
userdata = Customizer().custom_user_properties(userdata)
except BaseException as e:
log.error("Unable to customize user properties: {}", e)
if "email" in userdata:
userdata["email"] = userdata["email"].lower()
return userdata
def get_module_from_string(
modulestring: str,
exit_on_fail: bool = False) -> Optional[ModuleType]:
"""
Getting a module import
when your module is stored as a string in a variable
"""
try:
return import_module(modulestring)
except ModuleNotFoundError as e:
if exit_on_fail:
raise e
return None
except BaseException as e:
if exit_on_fail:
raise e
log.error("Module {} not found.\nError: {}", modulestring, e)
return None
def api(path, method, base="api", payload=None):
host = BotApiClient.variables.get("backend_host")
port = Env.get("FLASK_PORT")
url = f"http://{host}:{port}/{base}/{path}"
log.debug("Calling {} on {}", method, url)
try:
response = requests.request(method, url=url, data=payload)
out = response.json()
# Never raised during tests: how to test it?
except Exception as e: # pragma: no cover
log.error(f"API call failed: {e}")
raise RestApiException(str(e), status_code=500)
if response.status_code >= 300:
raise RestApiException(out, status_code=response.status_code)
return out
