class RequestState(db.Model): """Represents a state (historical or present) of a request.""" id = db.Column(db.Integer, primary_key=True) state = db.Column(db.Integer, nullable=False) state_reason = db.Column(db.String, nullable=False) updated = db.Column(db.DateTime(), nullable=False, default=sqlalchemy.func.now()) request_id = db.Column(db.Integer, db.ForeignKey("request.id"), index=True, nullable=False) request = db.relationship("Request", foreign_keys=[request_id], back_populates="states") @property def state_name(self): """Get the state's display name.""" if self.state: return RequestStateMapping(self.state).name def __repr__(self): return '<RequestState id={} state="{}" request_id={}>'.format( self.id, self.state_name, self.request_id)
class User(db.Model, UserMixin): """Represents an external user that owns a Cachito request.""" id = db.Column(db.Integer, primary_key=True) username = db.Column(db.String, index=True, unique=True, nullable=False) requests = db.relationship("Request", foreign_keys=[Request.user_id], back_populates="user") @classmethod def get_or_create(cls, username): """ Get the user from the database and create it if it doesn't exist. :param str username: the username of the user :return: a User object based on the input username; the User object will be added to the database session, but not committed, if it was created :rtype: User """ user = cls.query.filter_by(username=username).first() if not user: user = cls(username=username) db.session.add(user) return user
class RequestState(db.Model): id = db.Column(db.Integer, primary_key=True) state = db.Column(db.Integer, nullable=False) state_reason = db.Column(db.String, nullable=False) updated = db.Column(db.DateTime(), nullable=False, default=sqlalchemy.func.now()) request_id = db.Column(db.Integer, db.ForeignKey('request.id'), nullable=False) request = db.relationship('Request', back_populates='states') @property def state_name(self): """Get the state's display name.""" if self.state: return RequestStateMapping(self.state).name def __repr__(self): return '<RequestState id={} state="{}" request_id={}>'.format( self.id, self.state_name, self.request_id)
class Request(db.Model): """A Cachito user request.""" id = db.Column(db.Integer, primary_key=True) repo = db.Column(db.String, nullable=False) ref = db.Column(db.String, nullable=False) user_id = db.Column(db.Integer, db.ForeignKey('user.id')) submitted_by_id = db.Column(db.Integer, db.ForeignKey('user.id')) request_state_id = db.Column( db.Integer, db.ForeignKey('request_state.id'), index=True, unique=True, ) state = db.relationship('RequestState', foreign_keys=[request_state_id]) dependencies = db.relationship( 'Dependency', foreign_keys=[ RequestDependency.request_id, RequestDependency.dependency_id, ], secondary=RequestDependency.__table__, ) dependency_replacements = db.relationship( 'Dependency', foreign_keys=[ RequestDependency.request_id, RequestDependency.replaced_dependency_id, ], secondary=RequestDependency.__table__, ) packages = db.relationship('Package', secondary=RequestPackage.__table__) pkg_managers = db.relationship('PackageManager', secondary=request_pkg_manager_table, backref='requests') states = db.relationship( 'RequestState', foreign_keys='RequestState.request_id', back_populates='request', order_by='RequestState.updated', ) environment_variables = db.relationship( 'EnvironmentVariable', secondary=request_environment_variable_table, backref='requests', order_by='EnvironmentVariable.name') submitted_by = db.relationship('User', foreign_keys=[submitted_by_id]) user = db.relationship('User', foreign_keys=[user_id], back_populates='requests') flags = db.relationship('Flag', secondary=request_flag_table, backref='requests', order_by='Flag.name') def __repr__(self): return '<Request {0!r}>'.format(self.id) def add_dependency(self, dependency, replaced_dependency=None): """ Associate a dependency with this request if the association doesn't exist. This replaces the use of ``request.dependencies.append`` to be able to associate a dependency that is being replaced using the ``replaced_dependency`` keyword argument. Note that the association is added to the database session but not committed. :param Dependency dependency: a Dependency object :param Dependency replaced_dependency: an optional Dependency object to mark as being replaced by the input dependency for this request :raises ValidationError: if the dependency is already associated with the request, but replaced_dependency is different than what is already associated """ # If the ID is not set, then the dependency was just created and is not part of the # database's transaction buffer. if not dependency.id or (replaced_dependency and not replaced_dependency.id): # Send the changes queued up in SQLAlchemy to the database's transaction buffer. This # will genereate an ID that can be used for the mapping below. db.session.flush() mapping = RequestDependency.query.filter_by( request_id=self.id, dependency_id=dependency.id).first() if mapping: if mapping.replaced_dependency_id != getattr( replaced_dependency, 'id', None): raise ValidationError( f'The dependency {dependency.to_json()} can\'t have a new replacement set' ) return mapping = RequestDependency(request_id=self.id, dependency_id=dependency.id) if replaced_dependency: mapping.replaced_dependency_id = replaced_dependency.id db.session.add(mapping) @property def bundle_archive(self): """ Get the path to the request's bundle archive. :return: the path to the request's bundle archive :rtype: str """ cachito_bundles_dir = flask.current_app.config['CACHITO_BUNDLES_DIR'] return os.path.join(cachito_bundles_dir, f'{self.id}.tar.gz') @property def bundle_temp_files(self): """ Get the path to the request's temporary files used to create the bundle archive. :return: the path to the temporary files :rtype: str """ cachito_bundles_dir = flask.current_app.config['CACHITO_BUNDLES_DIR'] return os.path.join(cachito_bundles_dir, 'temp', str(self.id)) @property def dependencies_count(self): """ Get the total number of dependencies for a request. :return: the number of dependencies :rtype: int """ return db.session.query( sqlalchemy.func.count(RequestDependency.dependency_id)).filter( RequestDependency.request_id == self.id).scalar() @property def packages_count(self): """ Get the total number of packages for a request. :return: the number of packages :rtype: int """ return db.session.query( sqlalchemy.func.count(RequestPackage.package_id)).filter( RequestPackage.request_id == self.id).scalar() @property def replaced_dependency_mappings(self): """ Get the RequestDependency objects for the current request which contain a replacement. :return: a list of RequestDependency :rtype: list """ return (RequestDependency.query.filter_by(request_id=self.id).filter( RequestDependency.replaced_dependency_id.isnot(None)).all()) def to_json(self, verbose=True): pkg_managers = [ pkg_manager.to_json() for pkg_manager in self.pkg_managers ] # Use this list comprehension instead of a RequestState.to_json method to avoid including # redundant information about the request itself states = [{ 'state': RequestStateMapping(state.state).name, 'state_reason': state.state_reason, 'updated': state.updated.isoformat(), } for state in self.states] # Reverse the list since the latest states should be first states = list(reversed(states)) latest_state = states[0] user = None # If auth is disabled, there will not be a user associated with this request if self.user: user = self.user.username env_vars_json = OrderedDict(env_var.to_json() for env_var in self.environment_variables) rv = { 'id': self.id, 'repo': self.repo, 'ref': self.ref, 'pkg_managers': pkg_managers, 'user': user, 'environment_variables': env_vars_json, 'flags': [flag.to_json() for flag in self.flags], } if self.submitted_by: rv['submitted_by'] = self.submitted_by.username else: rv['submitted_by'] = None # Show the latest state information in the first level of the JSON rv.update(latest_state) if verbose: rv['state_history'] = states replacement_id_to_replacement = { replacement.id: replacement.to_json() for replacement in self.dependency_replacements } dep_id_to_replacement = { mapping.dependency_id: replacement_id_to_replacement[mapping.replaced_dependency_id] for mapping in self.replaced_dependency_mappings } rv['dependencies'] = [ dep.to_json(dep_id_to_replacement.get(dep.id), force_replaces=True) for dep in self.dependencies ] rv['packages'] = [package.to_json() for package in self.packages] else: rv['dependencies'] = self.dependencies_count rv['packages'] = self.packages_count return rv @classmethod def from_json(cls, kwargs): # Validate all required parameters are present required_params = {'repo', 'ref'} optional_params = { 'dependency_replacements', 'flags', 'pkg_managers', 'user' } missing_params = required_params - set(kwargs.keys()) - optional_params if missing_params: raise ValidationError('Missing required parameter(s): {}'.format( ', '.join(missing_params))) # Don't allow the user to set arbitrary columns or relationships invalid_params = set(kwargs.keys()) - required_params - optional_params if invalid_params: raise ValidationError( 'The following parameters are invalid: {}'.format( ', '.join(invalid_params))) request_kwargs = deepcopy(kwargs) # Validate package managers are correctly provided pkg_managers_names = request_kwargs.pop('pkg_managers', None) # If no package managers are specified, then Cachito will detect them automatically if pkg_managers_names: pkg_managers = PackageManager.get_pkg_managers(pkg_managers_names) request_kwargs['pkg_managers'] = pkg_managers flag_names = request_kwargs.pop('flags', None) if flag_names: flag_names = set(flag_names) found_flags = (Flag.query.filter(Flag.name.in_(flag_names)).filter( Flag.active).all()) if len(flag_names) != len(found_flags): found_flag_names = set(flag.name for flag in found_flags) invalid_flags = flag_names - found_flag_names raise ValidationError('Invalid/Inactive flag(s): {}'.format( ', '.join(invalid_flags))) request_kwargs['flags'] = found_flags dependency_replacements = request_kwargs.pop('dependency_replacements', []) if not isinstance(dependency_replacements, list): raise ValidationError('"dependency_replacements" must be an array') for dependency_replacement in dependency_replacements: Dependency.validate_replacement_json(dependency_replacement) submitted_for_username = request_kwargs.pop('user', None) # current_user.is_authenticated is only ever False when auth is disabled if submitted_for_username and not current_user.is_authenticated: raise ValidationError( 'Cannot set "user" when authentication is disabled') if current_user.is_authenticated: if submitted_for_username: allowed_users = flask.current_app.config[ 'CACHITO_USER_REPRESENTATIVES'] if current_user.username not in allowed_users: raise Forbidden( 'You are not authorized to create a request on behalf of another user' ) submitted_for = User.get_or_create(submitted_for_username) if not submitted_for.id: # Send the changes queued up in SQLAlchemy to the database's transaction buffer. # This will genereate an ID that can be used below. db.session.flush() request_kwargs['user_id'] = submitted_for.id request_kwargs['submitted_by_id'] = current_user.id else: request_kwargs['user_id'] = current_user.id request = cls(**request_kwargs) request.add_state('in_progress', 'The request was initiated') return request def add_state(self, state, state_reason): """ Add a RequestState associated with the current request. :param str state: the state name :param str state_reason: the reason explaining the state transition :raises ValidationError: if the state is invalid """ if self.state and self.state.state_name == 'stale' and state != 'stale': raise ValidationError('A stale request cannot change states') try: state_int = RequestStateMapping.__members__[state].value except KeyError: raise ValidationError( 'The state "{}" is invalid. It must be one of: {}.'.format( state, ', '.join(RequestStateMapping.get_state_names()))) request_state = RequestState(state=state_int, state_reason=state_reason) self.states.append(request_state) # Send the changes queued up in SQLAlchemy to the database's transaction buffer. # This will genereate an ID that can be used below. db.session.flush() self.request_state_id = request_state.id
class Request(db.Model): """A Cachito user request.""" id = db.Column(db.Integer, primary_key=True) repo = db.Column(db.String, nullable=False, index=True) ref = db.Column(db.String, nullable=False, index=True) user_id = db.Column(db.Integer, db.ForeignKey("user.id")) submitted_by_id = db.Column(db.Integer, db.ForeignKey("user.id")) request_state_id = db.Column(db.Integer, db.ForeignKey("request_state.id"), index=True, unique=True) packages_count = db.Column(db.Integer) dependencies_count = db.Column(db.Integer) state = db.relationship("RequestState", foreign_keys=[request_state_id]) pkg_managers = db.relationship("PackageManager", secondary=request_pkg_manager_table, backref="requests") states = db.relationship( "RequestState", foreign_keys="RequestState.request_id", back_populates="request", order_by="RequestState.updated", ) environment_variables = db.relationship( "EnvironmentVariable", secondary=request_environment_variable_table, backref="requests", order_by="EnvironmentVariable.name", ) submitted_by = db.relationship("User", foreign_keys=[submitted_by_id]) user = db.relationship("User", foreign_keys=[user_id], back_populates="requests") flags = db.relationship("Flag", secondary=request_flag_table, backref="requests", order_by="Flag.name") config_files_base64 = db.relationship( "ConfigFileBase64", secondary=request_config_file_base64_table, backref="requests") def __repr__(self): return "<Request {0!r}>".format(self.id) @property def content_manifest(self): """ Get the Image Content Manifest for a request. :return: the ContentManifest object for the request :rtype: ContentManifest """ packages_data = self._get_packages_data() packages = [ content_manifest.Package.from_json(package) for package in packages_data.packages ] return content_manifest.ContentManifest(self, packages) def _is_complete(self): if len(self.states) > 0: latest_state = self.states[-1] return latest_state.state_name == RequestStateMapping.complete.name return False def _get_packages_data(self): packages_data = PackagesData() if self._is_complete(): bundle_dir = RequestBundleDir( self.id, root=flask.current_app.config["CACHITO_BUNDLES_DIR"]) packages_data.load(bundle_dir.packages_data) return packages_data def to_json(self, verbose=True): """ Generate the JSON representation of the request. :param bool verbose: determines if the JSON should have verbose details :return: the JSON representation of the request :rtype: dict """ pkg_managers = [ pkg_manager.to_json() for pkg_manager in self.pkg_managers ] user = None # If auth is disabled, there will not be a user associated with this request if self.user: user = self.user.username env_vars_json = OrderedDict() for env_var in self.environment_variables: env_vars_json[env_var.name] = env_var.value rv = { "id": self.id, "repo": self.repo, "ref": self.ref, "pkg_managers": pkg_managers, "user": user, "environment_variables": env_vars_json, "flags": [flag.to_json() for flag in self.flags], } if self.submitted_by: rv["submitted_by"] = self.submitted_by.username else: rv["submitted_by"] = None def _state_to_json(state): return { "state": RequestStateMapping(state.state).name, "state_reason": state.state_reason, "updated": state.updated.isoformat(), } if verbose: rv["configuration_files"] = flask.url_for( "api_v1.get_request_config_files", request_id=self.id, _external=True) rv["content_manifest"] = flask.url_for( "api_v1.get_request_content_manifest", request_id=self.id, _external=True) rv["environment_variables_info"] = flask.url_for( "api_v1.get_request_environment_variables", request_id=self.id, _external=True) # Use this list comprehension instead of a RequestState.to_json method to avoid # including redundant information about the request itself states = [_state_to_json(state) for state in self.states] # Reverse the list since the latest states should be first states = list(reversed(states)) latest_state = states[0] rv["state_history"] = states packages_data = self._get_packages_data() rv["packages"] = packages_data.packages rv["dependencies"] = packages_data.all_dependencies dep: Dict[str, Any] for dep in itertools.chain( rv["dependencies"], (pkg_dep for pkg in rv["packages"] for pkg_dep in pkg["dependencies"]), ): dep.setdefault("replaces", None) if flask.current_app.config["CACHITO_REQUEST_FILE_LOGS_DIR"]: rv["logs"] = { "url": flask.url_for("api_v1.get_request_logs", request_id=self.id, _external=True) } else: latest_state = _state_to_json(self.state) rv["packages"] = self.packages_count rv["dependencies"] = self.dependencies_count # Show the latest state information in the first level of the JSON rv.update(latest_state) return rv @classmethod def from_json(cls, kwargs): """ Create a Request object from JSON. :param dict kwargs: the dictionary representing the request :return: the Request object :rtype: Request """ # Validate all required parameters are present required_params = {"repo", "ref"} optional_params = { "dependency_replacements", "flags", "packages", "pkg_managers", "user" } missing_params = required_params - set(kwargs.keys()) - optional_params if missing_params: raise ValidationError("Missing required parameter(s): {}".format( ", ".join(missing_params))) # Don't allow the user to set arbitrary columns or relationships invalid_params = set(kwargs.keys()) - required_params - optional_params if invalid_params: raise ValidationError( "The following parameters are invalid: {}".format( ", ".join(invalid_params))) if not is_request_ref_valid(kwargs["ref"]): raise ValidationError( 'The "ref" parameter must be a 40 character hex string') request_kwargs = deepcopy(kwargs) # Validate package managers are correctly provided pkg_managers_names = request_kwargs.pop("pkg_managers", None) # Default to the default package managers if pkg_managers_names is None: flask.current_app.logger.debug( "Using the default package manager(s) (%s) on the request", ", ".join(flask.current_app. config["CACHITO_DEFAULT_PACKAGE_MANAGERS"]), ) pkg_managers_names = flask.current_app.config[ "CACHITO_DEFAULT_PACKAGE_MANAGERS"] pkg_managers = PackageManager.get_pkg_managers(pkg_managers_names) request_kwargs["pkg_managers"] = pkg_managers _validate_request_package_configs(request_kwargs, pkg_managers_names or []) # Remove this from the request kwargs since it's not used as part of the creation of # the request object request_kwargs.pop("packages", None) flag_names = request_kwargs.pop("flags", None) if flag_names: flag_names = set(flag_names) found_flags = Flag.query.filter(Flag.name.in_(flag_names)).filter( Flag.active).all() if len(flag_names) != len(found_flags): found_flag_names = set(flag.name for flag in found_flags) invalid_flags = flag_names - found_flag_names raise ValidationError("Invalid/Inactive flag(s): {}".format( ", ".join(invalid_flags))) request_kwargs["flags"] = found_flags dependency_replacements = request_kwargs.pop("dependency_replacements", []) validate_dependency_replacements(dependency_replacements) submitted_for_username = request_kwargs.pop("user", None) # current_user.is_authenticated is only ever False when auth is disabled if submitted_for_username and not current_user.is_authenticated: raise ValidationError( 'Cannot set "user" when authentication is disabled') if current_user.is_authenticated: if submitted_for_username: allowed_users = flask.current_app.config[ "CACHITO_USER_REPRESENTATIVES"] if current_user.username not in allowed_users: flask.current_app.logger.error( "The user %s tried to submit a request on behalf of another user, but is " "not allowed", current_user.username, ) raise Forbidden( "You are not authorized to create a request on behalf of another user" ) submitted_for = User.get_or_create(submitted_for_username) request_kwargs["user"] = submitted_for request_kwargs["submitted_by"] = current_user else: request_kwargs["user"] = current_user._get_current_object() request = cls(**request_kwargs) request.add_state("in_progress", "The request was initiated") return request def add_state(self, state, state_reason): """ Add a RequestState associated with the current request. :param str state: the state name :param str state_reason: the reason explaining the state transition :raises ValidationError: if the state is invalid """ try: new_state: RequestStateMapping = RequestStateMapping[state] except KeyError: raise ValidationError( 'The state "{}" is invalid. It must be one of: {}.'.format( state, ", ".join(RequestStateMapping.get_state_names()))) if self.state: from_state_name: str = self.state.state_name from_state = RequestStateMapping[from_state_name] if not RequestStateMapping.allow_transition(from_state, new_state): raise ValidationError( f"State transition is not allowed from {from_state_name} to {state}." ) request_state = RequestState(state=new_state.value, state_reason=state_reason) self.states.append(request_state) # Send the changes queued up in SQLAlchemy to the database's transaction buffer. # This will generate an ID that can be used below. db.session.add(request_state) db.session.flush() self.state = request_state
class RequestState(db.Model): """Represents a state (historical or present) of a request.""" id = db.Column(db.Integer, primary_key=True) state = db.Column(db.Integer, nullable=False) state_reason = db.Column(db.String, nullable=False) updated = db.Column(db.DateTime(), nullable=False, default=sqlalchemy.func.now()) request_id = db.Column(db.Integer, db.ForeignKey("request.id"), index=True, nullable=False) request = db.relationship("Request", foreign_keys=[request_id], back_populates="states") @property def state_name(self): """Get the state's display name.""" if self.state: return RequestStateMapping(self.state).name def __repr__(self): return '<RequestState id={} state="{}" request_id={}>'.format( self.id, self.state_name, self.request_id) @classmethod def get_final_states_query(cls): """Return query that filters complete/failed states and includes a 'duration' column.""" # We enumerate states for each request and get 'updated' field of the following state. # It allows to find out time between two states. states = db.session.query( cls.request_id, cls.updated, func.lead(cls.updated, 1).over(partition_by=cls.request_id, order_by=cls.updated).label("next_updated"), func.row_number().over(partition_by=cls.request_id, order_by=cls.updated).label("num"), ).subquery() return ( db.session.query( cls.request_id, cls.state, cls.state_reason, cls.updated, func.extract( "epoch", cls.updated.cast(TIMESTAMP) - states.c.updated.cast(TIMESTAMP), ).label("duration"), func.extract( "epoch", states.c.next_updated.cast(TIMESTAMP) - states.c.updated.cast(TIMESTAMP), ).label("time_in_queue"), ).join(states, states.c.request_id == cls.request_id).filter( cls.state.in_([ RequestStateMapping.complete.value, RequestStateMapping.failed.value ])) # We need only 'init' state information here to join it with the final state. .filter(states.c.num == 1))
class User(db.Model, UserMixin): id = db.Column(db.Integer, primary_key=True) username = db.Column(db.String, unique=True, nullable=False) requests = db.relationship('Request', back_populates='user')
class Request(db.Model): """A Cachito user request.""" id = db.Column(db.Integer, primary_key=True) repo = db.Column(db.String, nullable=False) ref = db.Column(db.String, nullable=False) user_id = db.Column(db.Integer, db.ForeignKey('user.id')) dependencies = db.relationship('Dependency', secondary=request_dependency_table, backref='requests') pkg_managers = db.relationship('PackageManager', secondary=request_pkg_manager_table, backref='requests') states = db.relationship('RequestState', back_populates='request', order_by='RequestState.updated') user = db.relationship('User', back_populates='requests') def __repr__(self): return '<Request {0!r}>'.format(self.id) def to_json(self): pkg_managers = [ pkg_manager.to_json() for pkg_manager in self.pkg_managers ] # Use this list comprehension instead of a RequestState.to_json method to avoid including # redundant information about the request itself states = [{ 'state': RequestStateMapping(state.state).name, 'state_reason': state.state_reason, 'updated': state.updated.isoformat(), } for state in self.states] # Reverse the list since the latest states should be first states = list(reversed(states)) latest_state = states[0] user = None # If auth is disabled, there will not be a user associated with this request if self.user: user = self.user.username rv = { 'dependencies': [dep.to_json() for dep in self.dependencies], 'id': self.id, 'repo': self.repo, 'ref': self.ref, 'pkg_managers': pkg_managers, 'state_history': states, 'user': user, } # Show the latest state information in the first level of the JSON rv.update(latest_state) return rv @classmethod def from_json(cls, kwargs): # Validate all required parameters are present required_params = {'repo', 'ref', 'pkg_managers'} missing_params = required_params - set(kwargs.keys()) if missing_params: raise ValidationError('Missing required parameter(s): {}'.format( ', '.join(missing_params))) # Don't allow the user to set arbitrary columns or relationships invalid_params = kwargs.keys() - required_params if invalid_params: raise ValidationError( 'The following parameters are invalid: {}'.format( ', '.join(invalid_params))) request_kwargs = deepcopy(kwargs) # Validate package managers are correctly provided pkg_managers_names = request_kwargs.pop('pkg_managers', None) if not pkg_managers_names: raise ValidationError('At least one package manager is required') pkg_managers_names = set(pkg_managers_names) found_pkg_managers = (PackageManager.query.filter( PackageManager.name.in_(pkg_managers_names)).all()) if len(pkg_managers_names) != len(found_pkg_managers): found_pkg_managers_names = set( pkg_manager.name for pkg_manager in found_pkg_managers) invalid_pkg_managers = pkg_managers_names - found_pkg_managers_names raise ValidationError('Invalid package manager(s): {}'.format( ', '.join(invalid_pkg_managers))) request_kwargs['pkg_managers'] = found_pkg_managers # current_user.is_authenticated is only ever False when auth is disabled if current_user.is_authenticated: request_kwargs['user_id'] = current_user.id request = cls(**request_kwargs) request.add_state('in_progress', 'The request was initiated') return request def add_state(self, state, state_reason): """ Add a RequestState associated with the current request. :param str state: the state name :param str state_reason: the reason explaining the state transition :raises ValidationError: if the state is invalid """ try: state_int = RequestStateMapping.__members__[state].value except KeyError: raise ValidationError( 'The state "{}" is invalid. It must be one of: {}.'.format( state, ', '.join(RequestStateMapping.get_state_names()))) request_state = RequestState(state=state_int, state_reason=state_reason) self.states.append(request_state) @property def last_state(self): """ Get the last RequestState associated with the current request. :return: the last RequestState :rtype: RequestState """ return (RequestState.query.filter_by(request_id=self.id).order_by( RequestState.updated.desc(), RequestState.id.desc()).first())