def upload(self, subfolder: Path, force: bool = False) -> Response: if "file" not in request.files: raise BadRequest("No files specified") myfile = request.files["file"] if not myfile.filename: # pragma: no cover raise BadRequest("Invalid filename") if not self.allowed_file(myfile.filename): raise BadRequest("File extension not allowed") Uploader.validate_upload_folder(subfolder) if not subfolder.exists(): subfolder.mkdir(parents=True, exist_ok=True) fname = secure_filename(myfile.filename) abs_file = subfolder.joinpath(fname) log.info("File request for [{}]({})", myfile, abs_file) if abs_file.exists(): if not force: raise Conflict( f"File '{fname}' already exists, use force parameter to overwrite" ) abs_file.unlink() # Save the file try: myfile.save(abs_file) log.debug("Absolute file path should be '{}'", abs_file) except Exception as e: # pragma: no cover log.error(e) raise ServiceUnavailable( "Permission denied: failed to write the file") # Check exists - but it is basicaly a test that cannot fail... # The has just been uploaded! if not abs_file.exists(): # pragma: no cover raise ServiceUnavailable("Unable to retrieve the uploaded file") ######################## # ## Final response abs_file.chmod(DEFAULT_PERMISSIONS) # Default redirect is to 302 state, which makes client # think that response was unauthorized.... # see http://dotnet.dzone.com/articles/getting-know-cross-origin return EndpointResource.response( { "filename": fname, "meta": self.get_file_metadata(abs_file) }, code=200, )
def upload(self, subfolder: Optional[str] = None, force: bool = False) -> Response: if "file" not in request.files: raise BadRequest("No files specified") myfile = request.files["file"] # Check file extension? if not self.allowed_file(myfile.filename): raise BadRequest("File extension not allowed") # Check file name fname = secure_filename(myfile.filename) abs_file = Uploader.absolute_upload_file(fname, subfolder) log.info("File request for [{}]({})", myfile, abs_file) if os.path.exists(abs_file): if not force: raise BadRequest( f"File '{fname}' already exists, use force parameter to overwrite" ) os.remove(abs_file) log.debug("Already exists, forced removal") # Save the file try: myfile.save(abs_file) log.debug("Absolute file path should be '{}'", abs_file) except Exception: # pragma: no cover raise ServiceUnavailable( "Permission denied: failed to write the file") # Check exists - but it is basicaly a test that cannot fail... # The has just been uploaded! if not os.path.exists(abs_file): # pragma: no cover raise ServiceUnavailable("Unable to retrieve the uploaded file") ######################## # ## Final response # Default redirect is to 302 state, which makes client # think that response was unauthorized.... # see http://dotnet.dzone.com/articles/getting-know-cross-origin return EndpointResource.response( { "filename": fname, "meta": self.get_file_metadata(abs_file) }, code=200, )
def initialize_connection( self, expiration: int, verification: int, **kwargs: str ) -> T: # Create a new instance of itself obj = self.__class__() exceptions = obj.get_connection_exception() if exceptions is None: exceptions = (Exception,) try: obj = obj.connect(**kwargs) except exceptions as e: log.error("{} raised {}: {}", obj.name, e.__class__.__name__, e) raise ServiceUnavailable( { "Service Unavailable": "This service is temporarily unavailable, " "please retry in a few minutes" } ) obj.connection_time = datetime.now() obj.connection_verification_time = None if verification > 0: ver = obj.connection_time + timedelta(seconds=verification) obj.connection_verification_time = ver obj.connection_expiration_time = None if expiration > 0: exp = obj.connection_time + timedelta(seconds=expiration) obj.connection_expiration_time = exp return obj
class RabbitExt(Connector): def __init__(self) -> None: self.connection: Optional[pika.BlockingConnection] = None super().__init__() def get_connection_exception(self): # Includes: # AuthenticationError, # ProbableAuthenticationError, # ProbableAccessDeniedError, # ConnectionClosed... return ( AMQPConnectionError, # Includes failures in name resolution socket.gaierror, ) def connect(self, **kwargs): variables = self.variables.copy() # Beware, if you specify a user different by the default, # then the send method will fail to to PRECONDITION_FAILED because # the user_id will not pass the verification # Locally save self.variables + kwargs to be used in send() variables.update(kwargs) ssl_enabled = Env.to_bool(variables.get("ssl_enabled")) log.info("Connecting to the Rabbit (SSL = {})", ssl_enabled) if (host := variables.get("host")) is None: raise ServiceUnavailable("Missing hostname") if (user := variables.get("user")) is None: raise ServiceUnavailable("Missing credentials")
def post(self, username: str) -> Response: self.auth.verify_blocked_username(username) user = self.auth.get_user(username=username) # if user is None this endpoint does nothing but the response # remain the same to prevent any user guessing if user is not None: auth = Connector.get_authentication_instance() activation_token, payload = auth.create_temporary_token( user, auth.ACTIVATE_ACCOUNT) server_url = get_frontend_url() rt = activation_token.replace(".", "+") url = f"{server_url}/public/register/{rt}" sent = send_activation_link(user, url) if not sent: # pragma: no cover raise ServiceUnavailable("Error sending email, please retry") auth.save_token(user, activation_token, payload, token_type=auth.ACTIVATE_ACCOUNT) msg = ("We are sending an email to your email address where " "you will find the link to activate your account") return self.response(msg)
def test_exceptions(self) -> None: with pytest.raises(RestApiException) as e: raise BadRequest("test") assert e.value.status_code == 400 with pytest.raises(RestApiException) as e: raise Unauthorized("test") assert e.value.status_code == 401 with pytest.raises(RestApiException) as e: raise Forbidden("test") assert e.value.status_code == 403 with pytest.raises(RestApiException) as e: raise NotFound("test") assert e.value.status_code == 404 with pytest.raises(RestApiException) as e: raise Conflict("test") assert e.value.status_code == 409 with pytest.raises(RestApiException) as e: raise ServerError("test") assert e.value.status_code == 500 with pytest.raises(RestApiException) as e: raise ServiceUnavailable("test") assert e.value.status_code == 503
def __init__(self): self.commands = {} self.variables = Env.load_variables_group(prefix="telegram") if not self.variables.get("api_key"): # pragma: no cover raise ServiceUnavailable("Missing API KEY") self.updater = Updater( self.variables.get("api_key"), # Starting from v13 use_context is True by default # use_context=True, workers=Env.to_int(self.variables.get("workers"), default=1), ) # Inline keyboard callback self.updater.dispatcher.add_handler( CallbackQueryHandler(self.inline_keyboard_button)) # Errors self.updater.dispatcher.add_error_handler(self.error_callback) self.admins = Bot.get_ids(self.variables.get("admins")) if not self.admins: # pragma: no cover print_and_exit("No admin list") self.users = Bot.get_ids(self.variables.get("users")) self.api = BotApiClient(self.variables)
def connect(self, **kwargs: str) -> "FTPExt": variables = self.variables.copy() variables.update(kwargs) if (host := variables.get("host")) is None: # pragma: no cover raise ServiceUnavailable("Missing hostname")
def make_login(self, username: str, password: str, totp_code: Optional[str]) -> Tuple[str, Payload, User]: self.verify_blocked_username(username) try: user = self.get_user(username=username) except ValueError as e: # pragma: no cover # SqlAlchemy can raise the following error: # A string literal cannot contain NUL (0x00) characters. log.error(e) raise BadRequest("Invalid input received") except Exception as e: # pragma: no cover log.error("Unable to connect to auth backend\n[{}] {}", type(e), e) raise ServiceUnavailable("Unable to connect to auth backend") if user is None: self.register_failed_login(username, user=None) self.log_event( Events.failed_login, payload={"username": username}, user=user, ) raise Unauthorized("Invalid access credentials", is_warning=True) # Currently only credentials are allowed if user.authmethod != "credentials": # pragma: no cover raise BadRequest("Invalid authentication method") if not self.verify_password(password, user.password): self.log_event( Events.failed_login, payload={"username": username}, user=user, ) self.register_failed_login(username, user=user) raise Unauthorized("Invalid access credentials", is_warning=True) self.verify_user_status(user) if self.SECOND_FACTOR_AUTHENTICATION and not totp_code: raise AuthMissingTOTP() if totp_code: self.verify_totp(user, totp_code) # Token expiration is capped by the user expiration date, if set payload, full_payload = self.fill_payload(user, expiration=user.expiration) token = self.create_token(payload) self.save_login(username, user, failed=False) self.log_event(Events.login, user=user) return token, full_payload, user
def get_authentication_instance() -> BaseAuthentication: if not Connector._authentication_module: Connector._authentication_module = Connector.get_module( Connector.authentication_service, BACKEND_PACKAGE ) if not Connector._authentication_module: # pragma: no cover log.critical("{} not available", Connector.authentication_service) raise ServiceUnavailable("Authentication service not available") return Connector._authentication_module.Authentication()
def post(self, **kwargs: Any) -> Response: """ Register new user """ email = kwargs.get("email") user = self.auth.get_user(username=email) if user is not None: raise Conflict(f"This user already exists: {email}") password_confirm = kwargs.pop("password_confirm") if kwargs.get("password") != password_confirm: raise Conflict("Your password doesn't match the confirmation") if self.auth.VERIFY_PASSWORD_STRENGTH: check, msg = self.auth.verify_password_strength( kwargs.get("password"), None) if not check: raise Conflict(msg) kwargs["is_active"] = False user = self.auth.create_user(kwargs, [self.auth.default_role]) default_group = self.auth.get_group(name=DEFAULT_GROUP_NAME) self.auth.add_user_to_group(user, default_group) self.auth.save_user(user) self.log_event(self.events.create, user, kwargs) try: smtp_client = smtp.get_instance() if Env.get_bool("REGISTRATION_NOTIFICATIONS"): # Sending an email to the administrator title = get_project_configuration("project.title", default="Unkown title") subject = f"{title} New credentials requested" body = f"New credentials request from {user.email}" smtp_client.send(body, subject) send_activation_link(smtp_client, self.auth, user) except BaseException as e: # pragma: no cover self.auth.delete_user(user) raise ServiceUnavailable( f"Errors during account registration: {e}") return self.response( "We are sending an email to your email address where " "you will find the link to activate your account")
def send_password_reset_link(smtp, uri, title, reset_email): # Internal templating body: Optional[str] = f"Follow this link to reset your password: {uri}" html_body = get_html_template("reset_password.html", {"url": uri}) if html_body is None: log.warning("Unable to find email template") html_body = body body = None subject = f"{title} Password Reset" # Internal email sending c = smtp.send(html_body, subject, reset_email, plain_body=body) # it cannot fail during tests, because the email sending is mocked if not c: # pragma: no cover raise ServiceUnavailable("Error sending email, please retry")
def connect(self, **kwargs): variables = self.variables.copy() # Beware, if you specify a user different by the default, # then the send method will fail to to PRECONDITION_FAILED because # the user_id will not pass the verification # Locally save self.variables + kwargs to be used in send() variables.update(kwargs) ssl_enabled = Env.to_bool(variables.get("ssl_enabled")) log.info("Connecting to the Rabbit (SSL = {})", ssl_enabled) if (host := variables.get("host")) is None: raise ServiceUnavailable("Missing hostname")
def wait_socket( host: str, port: int, service_name: str, retries: int = DEFAULT_MAX_RETRIES ) -> None: SLEEP_TIME = 2 TIMEOUT = 1 log.debug("Waiting for {} ({}:{})", service_name, host, port) counter = 0 begin = time.time() while True: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.settimeout(TIMEOUT) try: result = s.connect_ex((host, port)) except socket.gaierror: result = errno.ESRCH if result == 0: log.info("Service {} is reachable", service_name) break counter += 1 if counter >= retries: t = math.ceil(time.time() - begin) raise ServiceUnavailable( f"{service_name} ({host}:{port}) unavailable after {t} seconds" ) if counter % 15 == 0: # pragma: no cover log.warning( "{} ({}:{}) is still unavailable after {} seconds", service_name, host, port, math.ceil(1 + time.time() - begin), ) else: log.debug("{} ({}:{}) not reachable", service_name, host, port) time.sleep(SLEEP_TIME)
def make_login(self, username: str, password: str) -> Tuple[str, Payload, User]: """ The method which will check if credentials are good to go """ try: user = self.get_user(username=username) except ValueError as e: # pragma: no cover # SqlAlchemy can raise the following error: # A string literal cannot contain NUL (0x00) characters. log.error(e) raise BadRequest("Invalid input received") except BaseException as e: # pragma: no cover log.error("Unable to connect to auth backend\n[{}] {}", type(e), e) raise ServiceUnavailable("Unable to connect to auth backend") if user is None: self.register_failed_login(username) self.log_event( Events.failed_login, payload={"username": username}, user=user, ) raise Unauthorized("Invalid access credentials", is_warning=True) # Check if Oauth2 is enabled if user.authmethod != "credentials": # pragma: no cover raise BadRequest("Invalid authentication method") # New hashing algorithm, based on bcrypt if self.verify_password(password, user.password): # Token expiration is capped by the user expiration date, if set payload, full_payload = self.fill_payload(user, expiration=user.expiration) token = self.create_token(payload) self.log_event(Events.login, user=user) return token, full_payload, user self.log_event( Events.failed_login, payload={"username": username}, user=user, ) self.register_failed_login(username) raise Unauthorized("Invalid access credentials", is_warning=True)
def post(self, reset_email: str) -> Response: reset_email = reset_email.lower() self.auth.verify_blocked_username(reset_email) user = self.auth.get_user(username=reset_email) if user is None: raise Forbidden( f"Sorry, {reset_email} is not recognized as a valid username", ) self.auth.verify_user_status(user) reset_token, payload = self.auth.create_temporary_token( user, self.auth.PWD_RESET) server_url = get_frontend_url() rt = reset_token.replace(".", "+") uri = Env.get("RESET_PASSWORD_URI", "/public/reset") complete_uri = f"{server_url}{uri}/{rt}" sent = send_password_reset_link(user, complete_uri, reset_email) if not sent: # pragma: no cover raise ServiceUnavailable("Error sending email, please retry") ################## # Completing the reset task self.auth.save_token(user, reset_token, payload, token_type=self.auth.PWD_RESET) msg = "We'll send instructions to the email provided if it's associated " msg += "with an account. Please check your spam/junk folder." self.log_event(self.events.reset_password_request, user=user) return self.response(msg)
def get_user(self, username: Optional[str] = None, user_id: Optional[str] = None) -> Optional[User]: try: if username: return self.db.User.query.filter_by(email=username).first() if user_id: return self.db.User.query.filter_by(uuid=user_id).first() except (sqlalchemy.exc.StatementError, sqlalchemy.exc.InvalidRequestError) as e: log.error(e) raise ServiceUnavailable("Backend database is unavailable") except ( sqlalchemy.exc.DatabaseError, sqlalchemy.exc.OperationalError, ) as e: # pragma: no cover raise e # only reached if both username and user_id are None return None
def get_channel(self): """ Return existing channel (if healthy) or create and return new one. :return: An healthy channel. :raises: AttributeError if the connection is None. """ if not self.connection: raise ServiceUnavailable(f"Service {self.name} is not available") if self.channel is None: log.debug("Creating new channel.") self.channel = self.connection.channel() self.channel.confirm_delivery() elif self.channel.is_closed: log.debug("Recreating channel.") self.channel = self.connection.channel() self.channel.confirm_delivery() return self.channel
class FTPExt(Connector): def __init__(self) -> None: self.connection: Union[FTP, FTP_TLS] = FTP() self.initialized = False super().__init__() # exception ftplib.error_reply # Exception raised when an unexpected reply is received from the server. # exception ftplib.error_temp # Exception raised when an error code signifying a temporary error # (response codes in the range 400–499) is received. # exception ftplib.error_perm # Exception raised when an error code signifying a permanent error # (response codes in the range 500–599) is received. # exception ftplib.error_proto # Exception raised when a reply is received from the server that does not fit the # response specifications of the File Transfer Protocol, # i.e. begin with a digit in the range 1–5. @staticmethod def get_connection_exception() -> ExceptionsList: return (socket.gaierror, ) def connect(self, **kwargs: str) -> "FTPExt": variables = self.variables.copy() variables.update(kwargs) if (host := variables.get("host")) is None: # pragma: no cover raise ServiceUnavailable("Missing hostname") if (user := variables.get("user")) is None: # pragma: no cover raise ServiceUnavailable("Missing credentials")
def test_exceptions(self) -> None: try: raise BadRequest("test") except RestApiException as e: assert e.status_code == 400 try: raise Unauthorized("test") except RestApiException as e: assert e.status_code == 401 try: raise Forbidden("test") except RestApiException as e: assert e.status_code == 403 try: raise NotFound("test") except RestApiException as e: assert e.status_code == 404 try: raise Conflict("test") except RestApiException as e: assert e.status_code == 409 try: raise ServerError("test") except RestApiException as e: assert e.status_code == 500 try: raise ServiceUnavailable("test") except RestApiException as e: assert e.status_code == 503
def post( self, name: str, surname: str, email: str, password: str, password_confirm: str, **kwargs: Any, ) -> Response: """Register new user""" user = self.auth.get_user(username=email) if user is not None: raise Conflict(f"This user already exists: {email}") if password != password_confirm: raise Conflict("Your password doesn't match the confirmation") check, msg = self.auth.verify_password_strength( pwd=password, old_pwd=None, email=email, name=name, surname=surname, ) if not check: raise Conflict(msg) kwargs["name"] = name kwargs["surname"] = surname kwargs["email"] = email kwargs["password"] = password kwargs["is_active"] = False user = self.auth.create_user(kwargs, [self.auth.default_role]) default_group = self.auth.get_group(name=DEFAULT_GROUP_NAME) self.auth.add_user_to_group(user, default_group) self.auth.save_user(user) self.log_event(self.events.create, user, kwargs) try: auth = Connector.get_authentication_instance() activation_token, payload = auth.create_temporary_token( user, auth.ACTIVATE_ACCOUNT ) server_url = get_frontend_url() rt = activation_token.replace(".", "+") log.debug("Activation token: {}", rt) url = f"{server_url}/public/register/{rt}" sent = send_activation_link(user, url) if not sent: # pragma: no cover raise ServiceUnavailable("Error sending email, please retry") auth.save_token( user, activation_token, payload, token_type=auth.ACTIVATE_ACCOUNT ) # Sending an email to the administrator if Env.get_bool("REGISTRATION_NOTIFICATIONS"): send_registration_notification(user) except Exception as e: # pragma: no cover self.auth.delete_user(user) raise ServiceUnavailable(f"Errors during account registration: {e}") return self.response( "We are sending an email to your email address where " "you will find the link to activate your account" )
def chunk_upload( self, upload_dir: Path, filename: str, chunk_size: Optional[int] = None) -> Tuple[bool, Response]: Uploader.validate_upload_folder(upload_dir) filename = secure_filename(filename) range_header = request.headers.get("Content-Range", "") total_length, start, stop = self.parse_content_range(range_header) if total_length is None or start is None or stop is None: raise BadRequest("Invalid request") completed = stop >= total_length # Default chunk size, put this somewhere if chunk_size is None: chunk_size = 1048576 file_path = upload_dir.joinpath(filename) # Uhm... this upload is not initialized? if not file_path.exists(): raise ServiceUnavailable( "Permission denied: the destination file does not exist") try: with open(file_path, "ab") as f: while True: chunk = request.stream.read(chunk_size) if not chunk: break f.seek(start) f.write(chunk) except PermissionError: raise ServiceUnavailable( "Permission denied: failed to write the file") if completed: file_path.chmod(DEFAULT_PERMISSIONS) return ( completed, EndpointResource.response( { "filename": filename, "meta": self.get_file_metadata(file_path) }, code=200, ), ) return ( completed, EndpointResource.response( "partial", headers={ "Access-Control-Expose-Headers": "Range", "Range": f"0-{stop - 1}", }, code=206, ), )
return (socket.gaierror, ) def connect(self, **kwargs: str) -> "FTPExt": variables = self.variables.copy() variables.update(kwargs) if (host := variables.get("host")) is None: # pragma: no cover raise ServiceUnavailable("Missing hostname") if (user := variables.get("user")) is None: # pragma: no cover raise ServiceUnavailable("Missing credentials") if (password := variables.get("password")) is None: # pragma: no cover raise ServiceUnavailable("Missing credentials") port = Env.get_int(variables.get("port"), 21) ssl_enabled = Env.to_bool(variables.get("ssl_enabled")) if ssl_enabled: context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) context.load_default_certs() # Disable certificate verification: # context.verify_mode = ssl.CERT_NONE # Enable certificate verification: context.verify_mode = ssl.CERT_REQUIRED context.check_hostname = False
def get_instance( self: T, verification: Optional[int] = None, expiration: Optional[int] = None, **kwargs: str, ) -> T: if not Connector.check_availability(self.name): raise ServiceUnavailable(f"Service {self.name} is not available") if verification is None: # this should be the default value for this connector verification = Env.to_int(self.variables.get("verification_time")) if expiration is None: # this should be the default value for this connector expiration = Env.to_int(self.variables.get("expiration_time")) # When context is empty this is a connection at loading time # Do not save it if stack.top is None: log.debug("First connection for {}", self.name) # can raise ServiceUnavailable exception obj = self.initialize_connection(expiration, verification, **kwargs) return obj unique_hash = str(sorted(kwargs.items())) obj = self.get_object(name=self.name, key=unique_hash) # if an expiration time is set, verify the instance age if obj and obj.connection_expiration_time: # the instance is invalidated if older than the expiration time if datetime.now() >= obj.connection_expiration_time: log.info("{} connection is expired", self.name) obj.disconnect() obj = None # If a verification time is set, verify the instance age if obj and obj.connection_verification_time: now = datetime.now() # the instance is verified if older than the verification time if now >= obj.connection_verification_time: # if the connection is still valid, set a new verification time if obj.is_connected(): # Set the new verification time ver = timedelta(seconds=verification) obj.connection_verification_time = now + ver # if the connection is no longer valid, invalidate the instance else: # pragma: no cover log.info( "{} is no longer connected, connector invalidated", self.name ) obj.disconnected = True # return the instance only if still connected # (and not invalidated by the verification check) if obj and not obj.disconnected: return obj # can raise ServiceUnavailable exception obj = self.initialize_connection(expiration, verification, **kwargs) self.set_object(name=self.name, obj=obj, key=unique_hash) return obj
def test_mongo(app: Flask) -> None: if not Connector.check_availability(CONNECTOR): try: obj = connector.get_instance() pytest.fail("No exception raised") # pragma: no cover except ServiceUnavailable: pass log.warning("Skipping {} tests: service not available", CONNECTOR) return None log.info("Executing {} tests", CONNECTOR) try: obj = connector.get_instance(host="invalidhostname", port=123) try: obj.Token.objects.first() except BaseException: raise ServiceUnavailable("") pytest.fail( "No exception raised on unavailable service") # pragma: no cover except ServiceUnavailable: pass obj = connector.get_instance() assert obj is not None try: obj.InvalidModel pytest.fail("No exception raised on InvalidModel") # pragma: no cover except AttributeError as e: assert str(e) == "Model InvalidModel not found" obj.disconnect() # a second disconnect should not raise any error obj.disconnect() # Create new connector with short expiration time obj = connector.get_instance(expiration=2, verification=1) obj_id = id(obj) # Connector is expected to be still valid obj = connector.get_instance(expiration=2, verification=1) assert id(obj) == obj_id time.sleep(1) # The connection should have been checked and should be still valid obj = connector.get_instance(expiration=2, verification=1) assert id(obj) == obj_id time.sleep(1) # Connection should have been expired and a new connector been created obj = connector.get_instance(expiration=2, verification=1) assert id(obj) != obj_id assert obj.is_connected() obj.disconnect() assert not obj.is_connected() # ... close connection again ... nothing should happens obj.disconnect() with connector.get_instance() as obj: assert obj is not None
raise except IntegrityError as e: message = str(e).split("\n") if error := parse_postgres_duplication_error(message): raise DatabaseDuplicatedEntry(error) if error := parse_mysql_duplication_error(message): raise DatabaseDuplicatedEntry(error) if error := parse_missing_error(message): raise DatabaseMissingRequiredProperty(error) # Should never happen except in case of a new alchemy version log.error("Unrecognized error message: {}", e) # pragma: no cover raise ServiceUnavailable("Duplicated entry") # pragma: no cover except InternalError as e: # pragma: no cover m = re.search( r"Incorrect string value: '(.*)' for column `.*`.`.*`.`(.*)` at row .*", str(e), ) if m: value = m.group(1) column = m.group(2) error = f"Invalid {column}: {value}" raise BadRequest(error) log.error("Unrecognized error message: {}", e)