def custom_init(self, pinit=False, pdestroy=False, abackend=None, **kwargs): """ Note: we ignore args here """ # recover instance with the parent method db = super().custom_init() if pdestroy: # massive destruction client = db.connection.database from pymongo import MongoClient client = MongoClient(self.variables.get('host'), int(self.variables.get('port'))) system_dbs = ['admin', 'local', 'config'] for db in client.database_names(): if db not in system_dbs: client.drop_database(db) log.critical("Dropped db '{}'", db) # if pinit: # pass return db
def remove(self, filename, subfolder=None, skip_response=False): """ Remove the file if requested """ abs_file = self.absolute_upload_file(filename, subfolder) # Check file existence if not os.path.exists(abs_file): log.critical("File '{}' not found", abs_file) return self.force_response( errors={"File missing": "File requested does not exists"}, code=hcodes.HTTP_BAD_NOTFOUND, ) # Remove the real file try: os.remove(abs_file) except Exception: log.critical("Cannot remove local file {}", abs_file) return self.force_response( errors={"Permissions": "Failed to remove file"}, code=hcodes.HTTP_DEFAULT_SERVICE_FAIL, ) log.warning("Removed '{}'", abs_file) if skip_response: return return self.force_response("Deleted", code=hcodes.HTTP_OK_BASIC)
def import_models(name: str, package: str, mandatory: bool = False) -> Dict[str, Type[Any]]: if package == BACKEND_PACKAGE: module_name = f"{package}.connectors.{name}.models" else: module_name = f"{package}.models.{name}" try: module = Meta.get_module_from_string(module_name, exit_on_fail=True) except BaseException as e: module = None if mandatory: log.critical(e) if not module: if mandatory: print_and_exit("Cannot load {} models from {}", name, module_name) return {} return Meta.get_new_classes_from_module(module)
def inline_keyboard_button( self, update: Update, context: CallbackContext[UD, CD, BD]) -> None: # pragma: no cover query = update.callback_query if not query: log.critical( "Debug code: query is empty in inline_keyboard_button") return None # Callback queries need to be answered, even if no notification to the user # is needed. Some clients may have trouble otherwise. # See https://core.telegram.org/bots/api#callbackquery query.answer() # The data cache contains the parameters wrapper of the command function. This # wrapper will be invoked by augmenting the original list of parameters with # the choice obtained from the inline keyboard argument if not query.data: log.critical( "Debug code: query.data is empty in inline_keyboard_button") return None data = data_cache.pop(query.data) query.edit_message_text(text=f"Selected option: {data['parameter']}") # func is the parameters wrapper of the command function func = data["func"] # original context args are augmented with the new choice data["context"].args.append(data["parameter"]) # Let's invoke the parameters wrapper with the new additional parameter func(data["update"], data["context"])
def print_and_exit(message: str, *args: Optional[str], **kwargs: Optional[str]) -> NoReturn: # Do not import outside the function to prevent circular imports from restapi.utilities.logs import log log.critical(message, *args, **kwargs) sys.exit(1)
def custom_init(self, pinit=False, pdestroy=False, abackend=None, **kwargs): """ Note: we ignore args here """ # recover instance with the parent method db = super().custom_init() # do init_app on the original flask sqlalchemy extension db.init_app(self.app) # careful on what you do with app context on sqlalchemy with self.app.app_context(): # check connection from sqlalchemy import text sql = text('SELECT 1') db.engine.execute(sql) if pdestroy: # massive destruction log.critical("Destroy current SQL data") db.drop_all() if pinit: # all is fine: now create table # because they should not exist yet db.create_all() return db
def custom_init(self, pinit=False, pdestroy=False, abackend=None, **kwargs): """ Note: we ignore args here """ # recover instance with the parent method graph = super().custom_init() # db.init_app(self.app) with self.app.app_context(): if pdestroy: log.critical("Destroy current Neo4j data") from neomodel import clear_neo4j_database clear_neo4j_database(graph.db) if pinit: auto_index = self.variables.get("autoindexing", 'True') == 'True' if auto_index: try: from neomodel import remove_all_labels, install_all_labels remove_all_labels() install_all_labels() except BaseException as e: log.exit(str(e)) return graph
def refresh_connection(self): if self.db.url is None: log.critical("Unable to refresh neo4j connection") return False log.info("Refreshing neo4j connection...") self.db.set_connection(self.db.url) return True
def destroy(self) -> None: graph = self.get_instance() if self.app: with self.app.app_context(): log.critical("Destroy current Neo4j data") clear_neo4j_database(graph.db)
def create_token(cls, payload: Payload) -> str: """ Generate a byte token with JWT library to encrypt the payload """ if cls.JWT_SECRET: return jwt.encode(payload, cls.JWT_SECRET, algorithm=cls.JWT_ALGO).decode( "ascii" ) else: # pragma: no cover log.critical("Server misconfiguration, missing jwt configuration") sys.exit(1)
def wrapper(*args: Any, **kwargs: Any) -> Any: try: return func(*args, **kwargs) except DatabaseDuplicatedEntry as e: # already catched and parser, raise up raise (e) except DoesNotExist as e: raise (e) except CypherSyntaxError as e: raise (e) except UniqueProperty as e: t = "already exists with label" m = re.search( rf"Node\([0-9]+\) {t} `(.+)` and property `(.+)` = '(.+)'", str(e)) if m: node = m.group(1) prop = m.group(2) val = m.group(3) error = f"A {node.title()} already exists with {prop}: {val}" raise DatabaseDuplicatedEntry(error) # Can't be tested, should never happen except in case of new neo4j version log.error("Unrecognized error message: {}", e) # pragma: no cover raise DatabaseDuplicatedEntry( "Duplicated entry") # pragma: no cover except RequiredProperty as e: # message = property 'xyz' on objects of class XYZ message = str(e) m = re.search(r"property '(.*)' on objects of class (.*)", str(e)) if m: missing_property = m.group(1) model = m.group(2) message = f"Missing property {missing_property} required by {model}" raise DatabaseMissingRequiredProperty(message) except DeflateError as e: log.warning(e) return None except ServiceUnavailable: # pragma: no cover # refresh_connection() raise # Catched in case of re-raise for example RequiredProperty -> BadRequest except RestApiException: # pragma: no cover raise except Exception as e: # pragma: no cover log.critical("Raised unknown exception: {}", type(e)) raise e
def manage_missing_parameter( self, func: Any, param: str, definition: fields.Field, update: Update, context: CallbackContext[UD, CD, BD], error: List[str], ) -> None: if not update.message: # pragma log.critical( "Debug code: missing message in manage_missing_parameter") return None # Parameters without description should raise some kind of errors/warnings? if "description" in definition.metadata: description = definition.metadata["description"] else: # pragma: no cover description = "???" # Enum -> InlineKeyboardButton if definition.validate and isinstance(definition.validate, validate.OneOf): choices = definition.validate.choices labels = definition.validate.labels if len(tuple(labels)) != len(tuple(choices)): labels = choices keyboard = [] for k, val in dict(zip(choices, labels)).items(): data_key = getUUID() # Because func, update and context are not (easily) serializable they # are saved in a data_cache and the callback will access them # by using the assigned unique data_key data_cache[data_key] = { "func": func, "update": update, "context": context, "parameter": k, } # All InlineKeyboardButton are registered with one single callback # function (SIGH and SOB!!). The data_key passed as callback_data will # be used to access the specific data and call again the command func # by augmenting the parameters list with the choice from the button keyboard.append( [InlineKeyboardButton(val, callback_data=data_key)]) reply_markup = InlineKeyboardMarkup(keyboard) update.message.reply_text(description, reply_markup=reply_markup) # Other errors # Never raised during tests else: # pragma: no cover update.message.reply_text(f"{description}\n{param}: {error[0]}")
def my_self(update: Update, context: CallbackContext[UD, CD, BD]) -> None: # Can't be true, since it is restricted_to_admins if not update.message or not update.message.from_user: # pragma: no cover log.critical("Error: user is missing") return None user_firstname = update.message.from_user.first_name user_id = update.message.from_user.id update.message.reply_text( f"Hello {user_firstname}, your Telegram ID is {user_id}")
def get_authentication_instance() -> BaseAuthentication: if not Connector._authentication_module: Connector._authentication_module = Connector.get_module( Connector.authentication_service, BACKEND_PACKAGE ) if not Connector._authentication_module: # pragma: no cover log.critical("{} not available", Connector.authentication_service) raise ServiceUnavailable("Authentication service not available") return Connector._authentication_module.Authentication()
def test_caching_autocleaning(self, client: FlaskClient) -> None: if Env.get_bool("AUTH_ENABLE"): headers, _ = self.do_login(client, None, None) else: headers = None # Syncronize this test to start at the beginning of the next second and # prevent the test to overlap a change of second # Since the caching is rounded to the second, few milliseconds cann make the # difference, for example: # A first request at 00:00:00.997 is cached # A second request at 00:00:01.002 is no longer cached, even if only 5 millisec # elapsed because the second changed # Added 0.01 just to avoid to exactly start at the beginning of the second t = 1.01 - datetime.now().microsecond / 1000000.0 log.critical("Sleeping {} sec", t) time.sleep(t) # the GET method is cached for 1 second # First response is not cached r = client.get(f"{API_URI}/tests/cache/short") assert r.status_code == 200 counter1 = self.get_content(r) # Second response is cached r = client.get(f"{API_URI}/tests/cache/short") assert r.status_code == 200 assert self.get_content(r) == counter1 # Third response is no longer cached time.sleep(1) r = client.get(f"{API_URI}/tests/cache/short") assert r.status_code == 200 counter2 = self.get_content(r) assert counter2 != counter1 # Fourth response is cached again r = client.get(f"{API_URI}/tests/cache/short") assert r.status_code == 200 assert self.get_content(r) == counter2 # Endpoint is unauthenticated, headers are ignored when building the cache key r = client.get(f"{API_URI}/tests/cache/short", headers=headers) assert r.status_code == 200 assert self.get_content(r) == counter2 # Tokens are ignored even if invalid r = client.get(f"{API_URI}/tests/cache/short", headers={"Authorization": "Bearer invalid"}) assert r.status_code == 200 assert self.get_content(r) == counter2
def wrapper(self, *args, **kwargs): out = None try: out = func(self, *args, **kwargs) # Catch the exception requested by the user except RestApiException as e: if e.is_warning: log.warning(e) else: log.exception(e) log.error(e) return self.response(e.args[0], code=e.status_code) except werkzeug.exceptions.BadRequest: # pragma: no cover # do not stop werkzeug BadRequest raise except werkzeug.exceptions.UnprocessableEntity: # do not stop werkzeug UnprocessableEntity, it will be # catched by handle_marshmallow_errors raise # raised in case of malformed Range header except werkzeug.exceptions.RequestedRangeNotSatisfiable: raise # Catch any other exception # errors with RabbitMQ credentials raised when sending Celery tasks except AccessRefused as e: # pragma: no cover log.critical(e) return self.response("Unexpected Server Error", code=500) except Exception as e: if SENTRY_URL is not None: # pragma: no cover capture_exception(e) excname = e.__class__.__name__ message = str(e) if not message: # pragma: no cover message = "Unknown error" log.exception(message) log.error("Catched {} exception: {}", excname, message) if excname in SYSTEM_EXCEPTIONS: return self.response( "Server failure; please contact admin.", code=400 ) return self.response({excname: message}, code=400) return out
def get(self, service): log.critical(detector.available_services) if not detector.check_availability(service): raise RestApiException( "Unknown service: {}".format(service), status_code=hcodes.HTTP_BAD_UNAUTHORIZED, ) service_instance = self.get_service_instance(service, global_instance=False) log.critical(service_instance) return self.force_response("Service is reachable: {}".format(service))
def get_bindings(self, exchange: str) -> Optional[List[Dict[str, str]]]: if not self.exchange_exists(exchange): log.critical("Does not exist") return None host = self.variables.get("host", "") schema = "" if not host.startswith("http"): if Env.to_bool(self.variables.get("ssl_enabled")): schema = "https://" else: schema = "http://" port = self.variables.get("management_port") # url-encode unsafe characters by also including / (thanks to safe parameter) # / -> %2F vhost = urllib.parse.quote(self.variables.get("vhost", "/"), safe="") user = self.variables.get("user") password = self.variables.get("password") # API Reference: # A list of all bindings in which a given exchange is the source. r = requests.get( f"{schema}{host}:{port}/api/exchanges/{vhost}/{exchange}/bindings/source", auth=HTTPBasicAuth(user, password), verify=False, ) response = r.json() if r.status_code != 200: # pragma: no cover raise RestApiException( {"RabbitMQ": response.get("error", "Unknown error")}, status_code=r.status_code, ) bindings = [] for row in response: # row == { # 'source': exchange-name, # 'vhost': probably '/', # 'destination': queue-or-dest-exchange-name, # 'destination_type': 'queue' or 'exchange', # 'routing_key': routing_key, # 'arguments': Dict, # 'properties_key': ?? as routing_key? # } bindings.append({ "exchange": row["source"], "routing_key": row["routing_key"], "queue": row["destination"], }) return bindings
def send_markdown(self, msg: str, update: Update) -> None: if not msg.strip(): # pragma: no cover return if update.message: self.updater.bot.send_message( chat_id=update.message.chat_id, text=msg.replace("_", "-"), parse_mode=ParseMode.MARKDOWN, ) else: # pragma: no cover log.critical( "Debug code: update.message in missing in send_markdown")
def get_class_from_string(classname, module, skip_error=False): """ Get a specific class from a module using a string variable """ myclass = None try: # Meta language for dinamically import myclass = getattr(module, classname) except AttributeError as e: if not skip_error: log.critical("Failed to load class from module: " + str(e)) else: pass return myclass
def destroy(self) -> None: instance = self.get_instance() if self.app: with self.app.app_context(): sql = text("SELECT 1") instance.db.engine.execute(sql) instance.db.session.remove() close_all_sessions() # massive destruction log.critical("Destroy current SQL data") instance.db.drop_all()
def destroy(self) -> None: instance = self.get_instance() # massive destruction client = instance.connection.database from pymongo import MongoClient uri = self._get_uri(self.variables) client = MongoClient(uri) system_dbs = ["admin", "local", "config"] for db in client.list_database_names(): if db not in system_dbs: client.drop_database(db) log.critical("Dropped db '{}'", db)
def wrapper(*args, **kwargs): # Recover the auth object auth_type, token = HTTPTokenAuth.get_authorization_token( allow_access_token_parameter=allow_access_token_parameter) # Internal API 'self' reference caller = Meta.get_self_reference_from_args(*args) if caller is None: # pragma: no cover # An exit here is really really dangerous, but even if # get_self_reference_from_args can return None, this case is quite # impossible... however with None the server can't continue! log.critical( "Server misconfiguration, self reference can't be None!" ) # with print_and_exit my-py does not understand that the execute # halts here... let's use an explicit exit sys.exit(1) if (auth_type is not None and auth_type == HTTPAUTH_SCHEME and request.method != "OPTIONS"): caller.unpacked_token = caller.auth.verify_token(token) # Check authentication. Optional authentication is valid if: # 1) token is missing # 2) token is valid # Invalid tokens are rejected if not caller.unpacked_token[0]: # Clear TCP receive buffer of any pending data _ = request.data # Mimic the response from a normal endpoint # To use the same standards # log.info("Invalid token received '{}'", token) log.debug("Invalid token received") return caller.response( "Invalid token received", headers=HTTPAUTH_ERR_HEADER, code=401, allow_html=True, ) request.environ[TOKEN_VALIDATED_KEY] = True return func(*args, **kwargs)
def make_login(self, username, password): """ The method which will check if credentials are good to go """ try: user = self.get_user_object(username=username) except BaseException as e: log.error("Unable to connect to auth backend\n[{}] {}", type(e), e) # log.critical("Please reinitialize backend tables") from restapi.exceptions import RestApiException raise RestApiException( "Unable to connect to auth backend", status_code=hcodes.HTTP_SERVER_ERROR, ) if user is None: return None, None try: # Check if Oauth2 is enabled if user.authmethod != 'credentials': return None, None except BaseException: # Missing authmethod as requested for authentication log.critical("Current authentication db models are broken!") return None, None # New hashing algorithm, based on bcrypt if self.verify_password(password, user.password): return self.create_token(self.fill_payload(user)) # old hashing; since 0.7.2. Removed me in a near future!! if self.check_old_password(user.password, password): log.warning( "Old password encoding for user {}, automatic convertion", user.email) now = datetime.now(pytz.utc) user.password = BaseAuthentication.get_password_hash(password) user.last_password_change = now self.save_user(user) return self.make_login(username, password) return None, None
def wrapper(self, *args, **kwargs): from neomodel.exceptions import RequiredProperty try: return func(self, *args, **kwargs) except DatabaseDuplicatedEntry as e: log.critical("boh") raise Conflict(str(e)) except RequiredProperty as e: log.critical("Missing required") raise BadRequest(e)
def invalid_message(self, update: Update, context: CallbackContext[UD, CD, BD]) -> None: if update.message: user = update.message.from_user.id if update.message.from_user else "N/A" log.info( "Received invalid message from {}: {}", user, update.message.text, ) if self.check_authorized(update, context): self.updater.bot.send_message( chat_id=update.message.chat_id, text="Invalid command, ask for /help", ) else: # pragma: no cover log.critical( "Debug code: update.message in missing in invalid_message")
def verify_roles( self, user: User, roles: Optional[List[Union[str, Role]]], required_roles: str = ALL_ROLES, warnings: bool = True, ) -> bool: if not roles: return True current_roles = self.get_roles_from_user(user) if required_roles == ALL_ROLES: for role in roles: if isinstance(role, Role): role = role.value if role not in current_roles: if warnings: log.warning("Auth role '{}' missing for request", role) return False return True if required_roles == ANY_ROLE: for role in roles: if isinstance(role, Role): role = role.value if role in current_roles: return True log.warning( "Expected at least one roles from {}, found none in {}", roles, current_roles, ) return False log.critical("Unknown role authorization requirement: {}", required_roles) return False
def change_password( self, user: User, password: str, new_password: Optional[str], password_confirm: Optional[str], ) -> bool: if new_password is None: raise BadRequest("Missing new password") if password_confirm is None: raise BadRequest("Missing password confirmation") if new_password != password_confirm: raise Conflict("Your password doesn't match the confirmation") check, msg = self.verify_password_strength( pwd=new_password, old_pwd=password, email=user.email, name=user.name, surname=user.surname, ) if not check: raise Conflict(msg) user.password = BaseAuthentication.get_password_hash(new_password) user.last_password_change = datetime.now(pytz.utc) self.save_user(user) self.log_event(Events.change_password, user=user) for token in self.get_tokens(user=user): try: self.invalidate_token(token=token["token"]) except Exception as e: # pragma: no cover log.critical("Failed to invalidate token {}", e) return True
def init_groups(self, force): create = False update = False default_group = self.get_group(name=DEFAULT_GROUP_NAME) # If there are no groups, let's create the default group if not self.get_groups(): create = True # If there are some groups skip group creation in absence of a force flag elif force: # If force flag is enable, create the default group if missing or update it create = default_group is None update = default_group is not None if create: default_group = self.create_group( { "shortname": DEFAULT_GROUP_NAME, "fullname": DEFAULT_GROUP_DESCR, } ) log.info("Injected default group") elif update: log.info("Default group already exists, updating") # Added to make the life easier to mypy... but cannot be False if default_group: default_group.shortname = DEFAULT_GROUP_NAME default_group.fullname = DEFAULT_GROUP_DESCR else: # pragma: no cover log.critical("Default group not found") self.save_group(default_group) elif default_group: log.info("Default group already exists") else: log.info("Default group does not exist but other groups do") return default_group
def check_authorized( self, update: Update, context: CallbackContext[UD, CD, BD], required_admin: bool = False, ) -> bool: if not update.message or not update.message.from_user: log.critical("Debug code: missing user in check_authorized") return False user = update.message.from_user user_id = user.id text = update.message.text if self.is_authorized(user_id, required_admin): log.info(f"User {user_id} requested: {text}") return True msg = "Unauthorized request!\n" for key, value in update.message.__dict__.items(): if key == "from_user": for k, v in value.__dict__.items(): if not k.startswith("_") and v is not None: msg += f"{k}: {v}\n" if key in ["date", "photo", "text"]: # print(key, value) if key == "text": msg += f"{key}: {value}\n" log.warning(msg) # Notify admins about violation self.admins_broadcast(msg) self.updater.bot.send_message( chat_id=update.message.chat_id, text= "Permission denied, you are not authorized to execute this command", ) return False