def _get_html_template(template_file: str, replaces: Dict[str, Any]) -> Optional[str]: # Custom templates from project backend/models/email/ template_path = CODE_DIR.joinpath(CUSTOM_PACKAGE, MODELS_DIR, "emails", template_file) if not template_path.exists(): # Core templates from restapi/connectors/smtp/templates/ template_path = ABS_RESTAPI_PATH.joinpath( CONNECTORS_FOLDER, "smtp", "templates", template_file, ) if not template_path.exists(): log.info("Template not found: {}", template_path) return None try: templateLoader = jinja2.FileSystemLoader( searchpath=template_path.parent) templateEnv = jinja2.Environment(loader=templateLoader, autoescape=True) template = templateEnv.get_template(template_file) replaces.setdefault("host", get_frontend_url()) return template.render(**replaces) except Exception as e: # pragma: no cover log.error("Error loading template {}: {}", template_file, e) return None
def refresh_token(self, jti): now = datetime.now() token_entry = self.db.Token.query.filter_by(jti=jti).first() if token_entry is None: return False if now > token_entry.expiration: self.invalidate_token(token=token_entry.token) log.info("This token is no longer valid: expired since {}", token_entry.strftime("%d/%m/%Y")) return False # Verify IP validity only after grace period is expired if token_entry.last_access + timedelta( seconds=self.grace_period) < now: ip = self.get_remote_ip() if token_entry.IP != ip: log.error( "This token is emitted for IP {}, invalid use from {}", token_entry.IP, ip) return False exp = now + timedelta(seconds=self.shortTTL) token_entry.last_access = now token_entry.expiration = exp try: self.db.session.add(token_entry) self.db.session.commit() except BaseException as e: log.error("DB error ({}), rolling back", e) self.db.session.rollback() return True
def test_01_GET_status(self, client): """ Test that the flask server is running and reachable """ # Check success endpoint = API_URI + '/status' alive_message = "Server is alive!" log.info("*** VERIFY if API is online") r = client.get(endpoint) assert r.status_code == hcodes.HTTP_OK_BASIC output = self.get_content(r) assert output == alive_message # Check failure log.info("*** VERIFY if invalid endpoint gives Not Found") r = client.get(API_URI) assert r.status_code == hcodes.HTTP_BAD_NOTFOUND # Check HTML response to status if agent/request is text/html from restapi.rest.response import MIMETYPE_HTML headers = {"Accept": MIMETYPE_HTML} r = client.get(endpoint, headers=headers) assert r.status_code == hcodes.HTTP_OK_BASIC output = r.data.decode('utf-8') assert output != alive_message assert alive_message in output assert "<html" in output assert "<body>" in output
def init(wait): """Initialize data for connected services""" if wait: mywait() log.info("Initialization requested") flask_cli({'name': 'Initializing services', 'init_mode': True})
def refresh_token(self, jti): now = datetime.now(pytz.utc) try: token_node = self.db.Token.nodes.get(jti=jti) if now > token_node.expiration: self.invalidate_token(token=token_node.token) log.info("This token is no longer valid: expired since {}", token_node.expiration.strftime("%d/%m/%Y")) return False # Verify IP validity only after grace period is expired if token_node.last_access + timedelta( seconds=self.grace_period) < now: ip = self.get_remote_ip() if token_node.IP != ip: log.error( "This token is emitted for IP {}, invalid use from {}", token_node.IP, ip) return False exp = now + timedelta(seconds=self.shortTTL) token_node.last_access = now token_node.expiration = exp token_node.save() return True except self.db.Token.DoesNotExist: log.warning("Token {} not found", jti) return False
def upload(self, subfolder: Path, force: bool = False) -> Response: if "file" not in request.files: raise BadRequest("No files specified") myfile = request.files["file"] if not myfile.filename: # pragma: no cover raise BadRequest("Invalid filename") if not self.allowed_file(myfile.filename): raise BadRequest("File extension not allowed") Uploader.validate_upload_folder(subfolder) if not subfolder.exists(): subfolder.mkdir(parents=True, exist_ok=True) fname = secure_filename(myfile.filename) abs_file = subfolder.joinpath(fname) log.info("File request for [{}]({})", myfile, abs_file) if abs_file.exists(): if not force: raise Conflict( f"File '{fname}' already exists, use force parameter to overwrite" ) abs_file.unlink() # Save the file try: myfile.save(abs_file) log.debug("Absolute file path should be '{}'", abs_file) except Exception as e: # pragma: no cover log.error(e) raise ServiceUnavailable( "Permission denied: failed to write the file") # Check exists - but it is basicaly a test that cannot fail... # The has just been uploaded! if not abs_file.exists(): # pragma: no cover raise ServiceUnavailable("Unable to retrieve the uploaded file") ######################## # ## Final response abs_file.chmod(DEFAULT_PERMISSIONS) # Default redirect is to 302 state, which makes client # think that response was unauthorized.... # see http://dotnet.dzone.com/articles/getting-know-cross-origin return EndpointResource.response( { "filename": fname, "meta": self.get_file_metadata(abs_file) }, code=200, )
def verify_token_validity(self, jti: str, user: User) -> bool: try: token_entry = self.db.Token.objects.raw({"jti": jti}).first() except self.db.Token.DoesNotExist: return False if token_entry.user_id is None or token_entry.user_id.email != user.email: return False now = datetime.now() if now > token_entry.expiration: self.invalidate_token(token=token_entry.token) log.info( "This token is no longer valid: expired since {}", token_entry.expiration.strftime("%d/%m/%Y"), ) return False # Verify IP validity only after grace period is expired if token_entry.creation + self.GRACE_PERIOD < now: ip = self.get_remote_ip() if token_entry.IP != ip: log.error( "This token is emitted for IP {}, invalid use from {}", token_entry.IP, ip, ) return False if token_entry.last_access + self.SAVE_LAST_ACCESS_EVERY < now: token_entry.last_access = now token_entry.save() return True
def unpack_token(self, token, raiseErrors=False): payload = None try: payload = jwt.decode(token, self.JWT_SECRET, algorithms=[self.JWT_ALGO]) # now > exp except jwt.exceptions.ExpiredSignatureError as e: # should this token be invalidated into the DB? if raiseErrors: raise e else: log.info("Unable to decode JWT token. {}", e) # now < nbf except jwt.exceptions.ImmatureSignatureError as e: if raiseErrors: raise e else: log.info("Unable to decode JWT token. {}", e) except Exception as e: if raiseErrors: raise e else: log.warning("Unable to decode JWT token. {}", e) return payload
def check_configuration(self): try: self.services_configuration = load_yaml_file( file='services.yaml', path=ABS_RESTAPI_CONFSPATH) except AttributeError as e: log.exit(e) for service in self.services_configuration: name, prefix = self.prefix_name(service) # Was this service enabled from the developer? enable_var = str(prefix + 'enable').upper() self.available_services[name] = self.get_bool_from_os(enable_var) if self.available_services[name]: # read variables variables = self.load_variables(service, enable_var, prefix) service['variables'] = variables # set auth service if name == self.authentication_name: self.authentication_service = variables.get('service') if self.authentication_service is None: log.info("No service defined for authentication") else: log.info( "Authentication based on '{}' service", self.authentication_service )
def __init__(self) -> None: # enter GeoData in neo4j attributes: Optional[List[str]] = None graph = neo4j.get_instance() with open(DATA_PATH.joinpath("geodata.tsv")) as fd: rd = csv.reader(fd, delimiter="\t", quotechar='"') for row in rd: if not attributes: # use the first row to get the list of attributes attributes = row else: props = dict(zip(attributes, row)) geodata = graph.GeoData.nodes.get_or_none( **{attributes[0]: row[0]}) if not geodata: # create a new one geodata = graph.GeoData(**props).save() else: # check if an update is needed for key, value in props.items(): if getattr(geodata, key) != value: setattr(geodata, key, value) geodata.save() log.info("GeoData nodes succesfully created")
def project_initialization(self, instances, app=None): """ Custom initialization of your project Please define your class Initializer in project/YOURPROJECT/backend/initialization/initialization.py """ try: # NOTE: this might be a pattern # see in meta.py:get_customizer_class module_path = "{}.{}.{}".format( CUSTOM_PACKAGE, 'initialization', 'initialization', ) module = Meta.get_module_from_string(module_path) meta = Meta() Initializer = meta.get_class_from_string( 'Initializer', module, skip_error=True ) if Initializer is None: log.debug("No custom init available") else: try: Initializer(instances, app=app) except BaseException as e: log.error("Errors during custom initialization: {}", e) else: log.info("Vanilla project has been initialized") except BaseException: log.debug("No custom init available")
def refresh_token(self, jti): try: token_entry = self.db.Token.objects.raw({'jti': jti}).first() except self.db.Token.DoesNotExist: return False now = datetime.now() if now > token_entry.expiration: self.invalidate_token(token=token_entry.token) log.info("This token is no longer valid: expired since {}", token_entry.strftime("%d/%m/%Y")) return False # Verify IP validity only after grace period is expired if token_entry.last_access + timedelta( seconds=self.grace_period) < now: ip = self.get_remote_ip() if token_entry.IP != ip: log.error( "This token is emitted for IP {}, invalid use from {}", token_entry.IP, ip) return False exp = now + timedelta(seconds=self.shortTTL) token_entry.last_access = now token_entry.expiration = exp token_entry.save() return True
def tests(wait, core, file, folder, destroy): # pragma: no cover """Compute tests and coverage""" if wait: while initializing(): log.debug("Waiting services initialization") time.sleep(5) mywait() num_opt = 0 if core: num_opt += 1 if file is not None: num_opt += 1 if folder is not None: num_opt += 1 if num_opt > 1: print_and_exit( "Please specify only one option between --core, --file and --folder" ) parameters = ["tests/tests.sh"] if core: parameters.append(current_package) else: parameters.append(CUSTOM_PACKAGE) if file is not None: # Can't be enabled due to mistral stuck at py38 # file = file.removeprefix("tests/") if file.startswith("tests/"): file = file[6:] if not os.path.isfile(os.path.join("tests", file)): print_and_exit("File not found: {}", file) parameters.append(file) elif folder is not None: if not os.path.isdir(os.path.join("tests", folder)): print_and_exit("Folder not found: {}", folder) parameters.append(folder) os.environ["TEST_CORE_ENABLED"] = str(core) # In prod mode tests are execute with the server running. # Destroy test fails with alchemy due to db locks if destroy and not PRODUCTION: os.environ["TEST_DESTROY_MODE"] = "1" try: log.info("Running tests... this may take some time") log.debug("Executing: {}", parameters) from plumbum import local command = local["bash"] command(parameters, stdout=sys.stdout, stderr=sys.stderr) sys.exit(0) except Exception as e: log.error(e) sys.exit(1)
def test_GET_status(self, client: FlaskClient) -> None: """Test that the flask server is running and reachable""" # Check success alive_message = "Server is alive" log.info("*** VERIFY if API is online") r = client.get(f"{API_URI}/status") assert r.status_code == 200 output = self.get_content(r) assert output == alive_message # Check failure log.info("*** VERIFY if invalid endpoint gives Not Found") r = client.get(API_URI) assert r.status_code == 404 if Env.get_bool("AUTH_ENABLE"): # Check /auth/status with no token or invalid token r = client.get(f"{AUTH_URI}/status") assert r.status_code == 401 r = client.get(f"{AUTH_URI}/status", headers={"Authorization": "Bearer ABC"}) assert r.status_code == 401 else: r = client.get(f"{AUTH_URI}/status") assert r.status_code == 404
def init_users_and_roles(self): # Handle system roles current_roles = [] current_roles_objs = self.db.Role.nodes.all() for role in current_roles_objs: current_roles.append(role.name) log.info("Current roles: {}", current_roles) for role in self.default_roles: if role not in current_roles: log.info("Creating role: {}", role) self.create_role(role) # Default user (if no users yet available) if not len(self.db.User.nodes) > 0: log.warning("No users inside graphdb. Injecting default.") self.create_user( { # 'uuid': getUUID(), 'email': self.default_user, # 'authmethod': 'credentials', 'name': 'Default', 'surname': 'User', 'password': self.default_password, }, roles=self.default_roles, ) else: log.debug("Users already created")
def unpack_token(cls, token: str, raiseErrors: bool = False) -> Optional[DecodedPayload]: try: return cast( DecodedPayload, jwt.decode(token, cls.JWT_SECRET, algorithms=[cls.JWT_ALGO]), ) # now > exp except ExpiredSignatureError as e: # should this token be invalidated into the DB? if raiseErrors: raise e else: log.info("Unable to decode JWT token. {}", e) # now < nbf except ImmatureSignatureError as e: if raiseErrors: raise e else: log.info("Unable to decode JWT token. {}", e) except Exception as e: if raiseErrors: raise e else: log.warning("Unable to decode JWT token. {}", e) return None
def send_file_streamed( filename: str, subfolder: Path, mime: Optional[str] = None, out_filename: Optional[str] = None, ) -> Response: Uploader.validate_upload_folder(subfolder) filename = secure_filename(filename) filepath = subfolder.joinpath(filename) if not filepath.is_file(): raise NotFound("The requested file does not exist") if mime is None: mime = Downloader.guess_mime_type(filepath) log.info("Providing streamed content from {} (mime={})", filepath, mime) response = Response( stream_with_context(Downloader.read_in_chunks(filepath)), mimetype=mime, ) if not out_filename: out_filename = filepath.name response.headers[ "Content-Disposition"] = f"attachment; filename={out_filename}" response.headers["Content-Length"] = filepath.stat().st_size return response
def test_04_logout(self, client: FlaskClient) -> None: """Check that you can logout with a valid token""" if not Env.get_bool("AUTH_ENABLE"): log.warning("Skipping logout tests") return # Check success log.info("*** VERIFY valid token") r = client.get(f"{AUTH_URI}/logout", headers=self.get("auth_header")) assert r.status_code == 204 events = self.get_last_events(2) assert events[0].event == Events.delete.value assert events[0].user == "-" assert events[0].target_type == "Token" assert events[0].url == "/auth/logout" assert events[1].event == Events.logout.value assert events[1].user == BaseAuthentication.default_user assert events[1].url == "/auth/logout" # Check failure log.info("*** VERIFY invalid token") r = client.get(f"{AUTH_URI}/logout") assert r.status_code == 401
def wrapper(self, *args, **kwargs): try: return func(self, *args, **kwargs) except BaseException: task_id = self.request.id task_name = self.request.task log.error("Celery task {} failed ({})", task_id, task_name) arguments = str(self.request.args) log.error("Failed task arguments: {}", arguments[0:256]) log.error("Task error: {}", traceback.format_exc()) if send_mail_is_active(): log.info("Sending error report by email", task_id, task_name) body = """ Celery task {} failed Name: {} Arguments: {} Error: {} """.format(task_id, task_name, str(self.request.args), traceback.format_exc()) project = get_project_configuration( "project.title", default='Unkown title', ) subject = "{}: task {} failed".format(project, task_name) send_mail(body, subject)
def sendmail(from_address, dest_addresses, msg): if from_address == "invalid1": raise SMTPException("SMTP Error") if from_address == "invalid2": raise BaseException("Generic Error") fpath = "/logs/mock.mail.lastsent.json" data = {"from": from_address, "cc": dest_addresses, "msg": msg} log.info("Mail mock sending email from {} to {}", from_address, dest_addresses) with open(fpath, "w+") as file: file.write(json.dumps(data)) log.info("Mail mock sent email from {} to {}", from_address, dest_addresses) log.info("Mail mock mail written in {}", fpath) log.info("Extracting body") fpath = "/logs/mock.mail.lastsent.body" b = email.message_from_string(msg) if b.is_multipart(): # get the first payload (the non html version) first_payload = b.get_payload()[0] payload = first_payload.get_payload() else: payload = b.get_payload() with open(fpath, "w+") as file: file.write(payload) log.info("Mail body written in {}", fpath)
def test_destroy() -> None: # Only executed if tests are run with --destroy flag if os.getenv("TEST_DESTROY_MODE", "0") != "1": log.info("Skipping destroy test, TEST_DESTROY_MODE not enabled") return # Always enable during core tests if not Connector.check_availability("authentication"): # pragma: no cover log.warning("Skipping authentication test: service not available") return # if Connector.check_availability("sqlalchemy"): # sql = sqlalchemy.get_instance() # # Close previous connections, otherwise the new create_app will hang # sql.session.remove() # sql.session.close_all() auth = Connector.get_authentication_instance() user = auth.get_user(username=BaseAuthentication.default_user) assert user is not None create_app(mode=ServerModes.DESTROY) try: auth = Connector.get_authentication_instance() user = auth.get_user(username=BaseAuthentication.default_user) assert user is None except ServiceUnavailable: pass
def unpack_token(cls, token: str, raiseErrors: bool = False) -> Optional[Payload]: try: if cls.JWT_SECRET: return jwt.decode(token, cls.JWT_SECRET, algorithms=[cls.JWT_ALGO]) else: print_and_exit( # pragma: no cover "Server misconfiguration, missing jwt configuration" ) # now > exp except ExpiredSignatureError as e: # should this token be invalidated into the DB? if raiseErrors: raise e else: log.info("Unable to decode JWT token. {}", e) # now < nbf except ImmatureSignatureError as e: if raiseErrors: raise e else: log.info("Unable to decode JWT token. {}", e) except Exception as e: if raiseErrors: raise e else: log.warning("Unable to decode JWT token. {}", e) return None
def store(self, chunk_size: int = 10000) -> None: self.close() log.info( "Storing {} ({})-[:{}]->({}) relationships", self.count, self.label1, self.relation, self.label2, ) field1 = self.fields[0].split(":")[0] field2 = self.fields[1].split(":")[0] properties: str = self.get_properties(self.fields[2:]) cypher = f""" LOAD CSV WITH HEADERS FROM 'file:///{self.filename}' AS line FIELDTERMINATOR '\t' CALL {{ WITH line MATCH (node1: {self.label1} {{{self.key1}: line.{field1}}}) MATCH (node2: {self.label2} {{{self.key2}: line.{field2}}}) MERGE (node1)-[:{self.relation} {{{properties}}}]->(node2) }} IN TRANSACTIONS OF {chunk_size} ROWS """ self.cypher_exec(cypher)
def check_authorized(self, update, context, required_admin=False): user = update.message.from_user user_id = user.id text = update.message.text if self.is_authorized(user_id, required_admin): log.info(f"User {user_id} requested: {text}") return True msg = "Unauthorized request!\n" for key, value in update.message.__dict__.items(): if key == "from_user": for k, v in value.__dict__.items(): if not k.startswith("_") and v is not None: msg += f"{k}: {v}\n" if key in ["date", "photo", "text"]: # print(key, value) if key == "text": msg += f"{key}: {value}\n" log.warning(msg) # Notify admins about violation self.admins_broadcast(msg) self.updater.bot.send_message( chat_id=update.message.chat_id, text= "Permission denied, you are not authorized to execute this command", ) return False
def start(self): self.updater.start_polling(read_latency=5) self.admins_broadcast("Bot is ready to accept requests") log.info("Bot is ready to accept requests") self.updater.idle() print_and_exit("Bot closed") # pragma: no cover
def decorator(func): log.info("Registering {}", cmd) self.updater.dispatcher.add_handler( CommandHandler(cmd, func, pass_args=True, run_async=run_async)) self.commands[cmd] = help return func
def read_in_streaming(self, absolute_path, headers=None): """ Reads obj from iRODS without saving a local copy """ log.info( "Downloading file {} in streaming with chunk size {}", absolute_path, self.chunk_size, ) try: obj = self.prc.data_objects.get(absolute_path) # NOTE: what about binary option? handle = obj.open('r') if headers is None: headers = {} return Response( stream_with_context( self.read_in_chunks(handle, self.chunk_size)), headers=headers, ) except iexceptions.DataObjectDoesNotExist: raise IrodsException( "This path does not exist or permission denied") except iexceptions.CollectionDoesNotExist: raise IrodsException( "This path does not exist or permission denied")
def project_init(options: Optional[Dict[str, bool]] = None) -> None: if Connector.authentication_service != NO_AUTH: authentication_instance = Connector.get_authentication_instance() connector_module = Connector.get_module( Connector.authentication_service, BACKEND_PACKAGE ) connector = connector_module.get_instance() log.debug("Initializing {}", Connector.authentication_service) connector.initialize() if options is None: options: Dict[str, bool] = {} with Connector.app.app_context(): authentication_instance.init_auth_db(options) log.info("Initialized authentication module") initializer = mem.initializer() if initializer: log.info("Vanilla project has been initialized") else: # pragma: no cover log.error("Errors during custom initialization") if TESTING: # Core test initialization initialize_testing_environment(authentication_instance) # Custom test initialization initializer.initialize_testing_environment()
def upload_chunked(self, destination, force=False, chunk_size=None): # Default chunk size, put this somewhere if chunk_size is None: chunk_size = 1048576 if os.path.exists(destination): log.warning("Already exists") if force: os.remove(destination) log.debug("Forced removal") else: log.error("File '{}' already exists", destination) return False with open(destination, "ab") as f: while True: chunk = request.stream.read(chunk_size) if not chunk: break f.write(chunk) # Check exists if not os.path.exists(destination): log.error("Unable to recover the uploaded file: {}", destination) return False log.info("File uploaded: {}", destination) return True
def refresh_connection(self): if self.db.url is None: log.critical("Unable to refresh neo4j connection") return False log.info("Refreshing neo4j connection...") self.db.set_connection(self.db.url) return True