class ClusterTowerJobEvent(ClusterEvent):
__mapper_args__ = {
'polymorphic_identity': ClusterEventType.TOWER_JOB,
}
tower_id = db.Column(db.Integer,
db.ForeignKey(tower_model.Server.id),
nullable=True)
#: ID of template in Tower.
tower_job_id = db.Column(db.Integer, nullable=True)
#: :type: :class:`ClusterStatus`
status = db.Column(db.Enum(ClusterStatus))
#: :type: :class:`rhub.tower.model.Server`
tower = db.relationship(tower_model.Server)
def to_dict(self):
data = super().to_dict()
data['status'] = self.status.value
return data
def get_tower_job_output(self, output_format='txt'):
"""
Create Tower client and get job stdout via API.
See :meth:`rhub.tower.client.Tower.template_job_stdout()`.
"""
tower_client = self.tower.create_tower_client()
return tower_client.template_job_stdout(self.tower_job_id,
output_format='txt')
class ClusterStatusChangeEvent(ClusterEvent):
__mapper_args__ = {
'polymorphic_identity': ClusterEventType.STATUS_CHANGE,
}
old_value = db.Column('status_old', db.Enum(ClusterStatus))
new_value = db.Column('status_new', db.Enum(ClusterStatus))
class ClusterReservationChangeEvent(ClusterEvent):
__mapper_args__ = {
'polymorphic_identity': ClusterEventType.RESERVATION_CHANGE,
}
old_value = db.Column('expiration_old',
db.DateTime(timezone=True),
nullable=True)
new_value = db.Column('expiration_new',
db.DateTime(timezone=True),
nullable=True)
class Quota(db.Model, ModelMixin):
__tablename__ = 'lab_quota'
id = db.Column(db.Integer, primary_key=True)
num_vcpus = db.Column(db.Integer, nullable=True)
ram_mb = db.Column(db.Integer, nullable=True)
num_volumes = db.Column(db.Integer, nullable=True)
volumes_gb = db.Column(db.Integer, nullable=True)
def to_dict(self):
data = super().to_dict()
del data['id']
return data
class SchedulerCronJob(db.Model, ModelMixin):
__tablename__ = 'scheduler_cron'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(128), unique=True, nullable=False)
description = db.Column(db.Text, default='', nullable=False)
enabled = db.Column(db.Boolean, default=True)
#: cron time expression, see man 5 crontab
time_expr = db.Column(db.String(128), nullable=False)
job_name = db.Column(db.Text, nullable=False)
job_params = db.Column(db.JSON, nullable=True)
last_run = db.Column(db.DateTime(timezone=True), nullable=True)
@validates('time_expr')
def validate_time_expr(self, key, value):
if not CronValidator.parse(value):
raise ValueError(f'Cron time expression {value!r} is not valid')
return value
@validates('job_name')
def validate_job_name(self, key, value):
if value not in jobs.CronJob.get_jobs():
raise ValueError('CronJob is not defined')
return value
@property
def job(self):
"""Get job instance :class:`jobs.CronJob`."""
return jobs.CronJob.get_jobs()[self.job_name]
class RegionProduct(db.Model, ModelMixin):
"""Region-Product N-N association table."""
__tablename__ = 'lab_region_product'
region_id = db.Column(db.Integer,
db.ForeignKey('lab_region.id'),
primary_key=True)
region = db.relationship('Region', back_populates='products_relation')
product_id = db.Column(db.Integer,
db.ForeignKey('lab_product.id'),
primary_key=True)
product = db.relationship('Product', back_populates='regions_relation')
enabled = db.Column(db.Boolean, default=True)
class Policy(db.Model, ModelMixin):
__tablename__ = 'policies'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(256), nullable=False)
department = db.Column(db.Text, nullable=False)
constraint_sched_avail = db.Column(db.ARRAY(db.Text), nullable=True)
constraint_serv_avail = db.Column(db.Numeric, nullable=True)
constraint_limit = db.Column(db.JSON, nullable=True)
constraint_density = db.Column(db.Text, nullable=True)
constraint_tag = db.Column(db.ARRAY(db.Text), nullable=True)
constraint_cost = db.Column(db.Numeric, nullable=True)
constraint_location = db.Column(db.Text, nullable=True)
def to_dict(self):
data = {}
data['constraint'] = {}
for column in self.__table__.columns:
key = column.name
value = getattr(self, column.name)
if 'constraint_' in key:
data['constraint'][key[11:]] = value
else:
data[key] = value
return data
@staticmethod
def flatten_data(data):
"""
Flatten constraint field in given JSON
"""
new = {}
for key, value in data.items():
if key == 'constraint':
for key, value in value.items():
new['constraint_' + key] = value
else:
new[key] = value
return new
@classmethod
def from_dict(cls, data):
return super().from_dict(cls.flatten_data(data))
def update_from_dict(self, data):
return super().update_from_dict(self.flatten_data(data))
class Server(db.Model, ModelMixin):
__tablename__ = 'tower_server'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(32), unique=True, nullable=False)
description = db.Column(db.Text, default='', nullable=False)
enabled = db.Column(db.Boolean, default=True)
url = db.Column(db.String(256), nullable=False)
verify_ssl = db.Column(db.Boolean, default=True)
#: Tower credentials path (Vault mount/path)
credentials = db.Column(db.String(256), nullable=False)
def create_tower_client(self) -> Tower:
"""
Create Tower client.
:returns: :class:`rhub.tower.client.Tower`
:raises: `RuntimeError` if failed to create client due to missing
credentials in vault
:raises: `Exception` any other errors
"""
vault = di.get(Vault)
credentials = vault.read(self.credentials)
if not credentials:
raise RuntimeError(
f'Missing credentials in vault; {vault!r} {self.credentials}')
return Tower(
url=self.url,
username=credentials['username'],
password=credentials['password'],
verify_ssl=self.verify_ssl,
)
class Job(db.Model, ModelMixin):
__tablename__ = 'tower_job'
id = db.Column(db.Integer, primary_key=True)
#: Reference to Tower template (:attr:`Template.id`).
template_id = db.Column(db.Integer,
db.ForeignKey('tower_template.id'),
nullable=False)
#: ID of job in Tower.
tower_job_id = db.Column(db.Integer, nullable=False)
#: UUID of user who launched job.
#: See: :meth:`rhub.auth.keycloak.KeycloakClient.user_get`.
#:
#: :type: `str`
launched_by = db.Column(postgresql.UUID, nullable=False, index=True)
#: :type: :class:`Template`
template = db.relationship('Template')
@property
def server(self):
return self.template.server
class ClusterHost(db.Model, ModelMixin):
__tablename__ = 'lab_cluster_host'
id = db.Column(db.Integer, primary_key=True)
cluster_id = db.Column(db.Integer,
db.ForeignKey('lab_cluster.id'),
nullable=False)
fqdn = db.Column(db.String(256), nullable=False)
ipaddr = db.Column(db.ARRAY(postgresql.INET))
num_vcpus = db.Column(db.Integer, nullable=True)
ram_mb = db.Column(db.Integer, nullable=True)
num_volumes = db.Column(db.Integer, nullable=True)
volumes_gb = db.Column(db.Integer, nullable=True)
#: :type: :class:`Cluster`
cluster = db.relationship('Cluster', back_populates='hosts')
class ClusterEvent(db.Model, ModelMixin):
__tablename__ = 'lab_cluster_event'
__mapper_args__ = {
'polymorphic_on': 'type',
}
id = db.Column(db.Integer, primary_key=True)
type = db.Column(db.Enum(ClusterEventType))
date = db.Column(db.DateTime(timezone=True))
user_id = db.Column(postgresql.UUID, nullable=True)
cluster_id = db.Column(db.Integer,
db.ForeignKey('lab_cluster.id'),
nullable=False)
#: :type: :class:`Cluster`
cluster = db.relationship('Cluster', back_populates='events')
@property
def user_name(self):
if self.user_id:
return di.get(KeycloakClient).user_get(self.user_id)['username']
return None
def to_dict(self):
data = {}
for column in self.__table__.columns:
if hasattr(self, column.name):
data[column.name] = getattr(self, column.name)
# These attributes have different column names, see subclasses below.
for i in ['old_value', 'new_value']:
if hasattr(self, i):
data[i] = getattr(self, i)
data['user_name'] = self.user_name
return data
class Template(db.Model, ModelMixin):
__tablename__ = 'tower_template'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(256), unique=True, nullable=False)
description = db.Column(db.Text, default='', nullable=False)
#: Reference to Tower server (:attr:`Server.id`).
server_id = db.Column(db.Integer,
db.ForeignKey('tower_server.id'),
nullable=False)
#: ID of template in Tower.
tower_template_id = db.Column(db.Integer, nullable=False)
#: Is template workflow?
tower_template_is_workflow = db.Column(db.Boolean, nullable=False)
#: :type: list of :class:`Job`
jobs = db.relationship('Job', back_populates='template')
#: :type: :class:`Server`
server = db.relationship('Server')
class Product(db.Model, ModelMixin):
__tablename__ = 'lab_product'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(64), unique=True, nullable=False)
description = db.Column(db.Text, nullable=False, default='')
enabled = db.Column(db.Boolean, default=True)
tower_template_name_create = db.Column(db.String(128), nullable=False)
tower_template_name_delete = db.Column(db.String(128), nullable=False)
parameters = db.Column(db.JSON, nullable=False)
flavors = db.Column(db.JSON, nullable=True)
#: :type: list of :class:`RegionProduct`
regions_relation = db.relationship('RegionProduct',
back_populates='product',
lazy='dynamic')
#: :type: list of :class:`Cluster`
clusters = db.relationship('Cluster', back_populates='product')
@property
def parameters_variables(self):
return [param['variable'] for param in self.parameters]
@property
def parameters_defaults(self):
return {
param['variable']: param['default']
for param in self.parameters if 'default' in param
}
def validate_cluster_params(self, cluster_params):
invalid_params = {}
if extra_params := set(cluster_params) - set(
self.parameters_variables):
for i in extra_params:
invalid_params[i] = 'not allowed'
for param_spec in self.parameters:
var = param_spec['variable']
if var not in cluster_params:
if param_spec['required']:
invalid_params[var] = 'is required'
continue
t = param_spec['type']
if t == 'string':
if not isinstance(cluster_params[var], str):
invalid_params[var] = 'must be a string'
if param_spec.get('maxLength') is not None:
max_len = param_spec['maxLength']
if len(cluster_params[var]) > max_len:
invalid_params[var] = (
f'invalid value, maximal length is {max_len}')
if param_spec.get('minLength') is not None:
min_len = param_spec['minLength']
if len(cluster_params[var]) < min_len:
invalid_params[var] = (
f'invalid vlue, minimal length is {min_len}')
elif t == 'integer':
if type(cluster_params[var]) is not int:
invalid_params[var] = 'must be an integer'
if param_spec.get('max') is not None:
max_val = param_spec['max']
if cluster_params[var] > max_val:
invalid_params[var] = (
f'invalid value, maximal value is {max_val}')
if param_spec.get('min') is not None:
min_val = param_spec['min']
if cluster_params[var] < min_val:
invalid_params[var] = (
f'invalid value, minimal value is {min_val}')
elif t == 'boolean':
if not isinstance(cluster_params[var], bool):
invalid_params[var] = 'must be a boolean'
if (e := param_spec.get('enum')) is not None:
if cluster_params[var] not in e:
invalid_params[var] = 'value not allowed'
class Cluster(db.Model, ModelMixin):
__tablename__ = 'lab_cluster'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(32), unique=True, nullable=False)
description = db.Column(db.Text, nullable=False, default='')
user_id = db.Column(postgresql.UUID, nullable=False)
group_id = db.Column(postgresql.UUID, nullable=True)
created = db.Column(db.DateTime(timezone=True))
region_id = db.Column(db.Integer,
db.ForeignKey('lab_region.id'),
nullable=False)
#: :type: :class:`Region`
region = db.relationship('Region', back_populates='clusters')
reservation_expiration = db.Column(db.DateTime(timezone=True),
nullable=True)
#: Cluster lifespan expiration (hard-limit), see
#: :meth:`Region.create_cluster`.
lifespan_expiration = db.Column(db.DateTime(timezone=True), nullable=True)
#: :type: :class:`ClusterStatus`
status = db.Column(db.Enum(ClusterStatus), nullable=True, default=None)
#: :type: list of :class:`ClusterEvent`
events = db.relationship('ClusterEvent',
back_populates='cluster',
cascade='all,delete-orphan')
#: :type: list of :class:`ClusterHost`
hosts = db.relationship('ClusterHost',
back_populates='cluster',
cascade='all,delete-orphan')
product_id = db.Column(db.Integer,
db.ForeignKey('lab_product.id'),
nullable=False)
product_params = db.Column(db.JSON, nullable=False)
product = db.relationship('Product', back_populates='clusters')
RESERVED_NAMES = [
'localhost',
'all',
'ungrouped',
'lab',
'cluster',
'region',
'tower',
]
@validates('name')
def validate_name(self, key, value):
if value.lower() in self.RESERVED_NAMES:
raise ValueError(f'{value!r} is reserved name')
if len(value) < 6:
raise ValueError('Cluster name is too short')
if len(value) > 20:
raise ValueError('Cluster name is too long')
if not re.match(r'^[0-9a-z]+$', value):
raise ValueError(
'Cluster name contains invalid characters, '
'allowed are only 0-9 and a-z (uppercase characters are not allowed).'
)
return value
@property
def quota(self):
"""
User quota.
:type: :class:`Quota` or `None`
"""
if self.region:
return self.region.user_quota
return None
@property
def quota_usage(self):
"""
User quota usage.
:type: dict or `None`
"""
usage = dict.fromkeys(
['num_vcpus', 'ram_mb', 'num_volumes', 'volumes_gb'], 0)
for host in self.hosts:
for k in usage:
usage[k] += getattr(host, k)
return usage
@property
def user_name(self):
return di.get(KeycloakClient).user_get(self.user_id)['username']
@property
def group_name(self):
if self.group_id:
return di.get(KeycloakClient).group_get(self.group_id)['name']
return None
@property
def shared(self):
return self.group_name == SHAREDCLUSTER_GROUP
@property
def tower_launch_extra_vars(self):
rhub_extra_vars = {
'rhub_cluster_id': self.id,
'rhub_cluster_name': self.name,
'rhub_product_id': self.product.id,
'rhub_product_name': self.product.name,
'rhub_region_id': self.region.id,
'rhub_region_name': self.region.name,
'rhub_user_id': self.user_id,
'rhub_user_name': self.user_name,
}
return rhub_extra_vars | self.product_params
def to_dict(self):
data = super().to_dict()
data['region_name'] = self.region.name
data['user_name'] = self.user_name
data['group_name'] = self.group_name
data['shared'] = self.shared
if self.hosts:
data['hosts'] = [host.to_dict() for host in self.hosts]
else:
data['hosts'] = []
if self.quota:
data['quota'] = self.quota.to_dict()
data['quota_usage'] = self.quota_usage
else:
data['quota'] = None
data['quota_usage'] = None
if self.status:
data['status'] = self.status.value
else:
data['status'] = None
data['product_name'] = self.product.name
return data
class Region(db.Model, ModelMixin):
__tablename__ = 'lab_region'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(32), unique=True, nullable=False)
location = db.Column(db.String(32), index=True, nullable=True)
description = db.Column(db.Text, nullable=False, default='')
banner = db.Column(db.Text, nullable=False, default='')
enabled = db.Column(db.Boolean, default=True)
user_quota_id = db.Column(db.Integer,
db.ForeignKey('lab_quota.id'),
nullable=True)
#: :type: :class:`Quota` resources limit that single user can use, if
#: exceeded the user should not be able to create new reservations
user_quota = db.relationship('Quota',
uselist=False,
foreign_keys=[user_quota_id])
total_quota_id = db.Column(db.Integer,
db.ForeignKey('lab_quota.id'),
nullable=True)
#: :type: :class:`Quota` total region resources limits, if exceeded no user
#: should be able to create new reservations
total_quota = db.relationship('Quota',
uselist=False,
foreign_keys=[total_quota_id])
lifespan_length = db.Column(db.Integer, nullable=True)
reservations_enabled = db.Column(db.Boolean, default=True)
reservation_expiration_max = db.Column(db.Integer, nullable=True)
owner_group = db.Column(postgresql.UUID, nullable=False)
#: Limits use only to specific group of people. `NULL` == shared lab.
users_group = db.Column(postgresql.UUID, nullable=True, index=True)
... # TODO policies?
tower_id = db.Column(db.Integer, db.ForeignKey(tower_model.Server.id))
#: :type: :class:`rhub.tower.model.Server`
tower = db.relationship(tower_model.Server)
openstack_url = db.Column(db.String(256), nullable=False)
#: OpenStack credentials path in Vault
openstack_credentials = db.Column(db.String(256), nullable=False)
openstack_project = db.Column(db.String(64), nullable=False)
openstack_domain_name = db.Column(db.String(64), nullable=False)
openstack_domain_id = db.Column(db.String(64), nullable=False)
#: Network providers that can be used in the region
openstack_networks = db.Column(db.ARRAY(db.String(64)), nullable=False)
#: SSH key name
openstack_keyname = db.Column(db.String(64), nullable=False)
satellite_hostname = db.Column(db.String(256), nullable=False)
satellite_insecure = db.Column(db.Boolean, default=False, nullable=False)
#: Satellite credentials path in Vault
satellite_credentials = db.Column(db.String(256), nullable=False)
dns_server_hostname = db.Column(db.String(256), nullable=False)
dns_server_zone = db.Column(db.String(256), nullable=False)
#: DNS server credentials path in Vault
dns_server_key = db.Column(db.String(256), nullable=False)
vault_server = db.Column(db.String(256), nullable=False)
download_server = db.Column(db.String(256), nullable=False)
#: :type: list of :class:`Cluster`
clusters = db.relationship('Cluster', back_populates='region')
#: :type: list of :class:`RegionProduct`
products_relation = db.relationship('RegionProduct',
back_populates='region',
lazy='dynamic')
_INLINE_CHILDS = ['openstack', 'satellite', 'dns_server']
@property
def lifespan_enabled(self):
return self.lifespan_length is not None
@property
def lifespan_delta(self):
""":type: :class:`datetime.timedelta` or `None`"""
if not self.lifespan_length:
return None
return datetime.timedelta(days=self.lifespan_length)
@property
def reservation_expiration_max_delta(self):
""":type: :class:`datetime.timedelta` or `None`"""
if not self.reservation_expiration_max:
return None
return datetime.timedelta(days=self.reservation_expiration_max)
@property
def owner_group_name(self):
return di.get(KeycloakClient).group_get(self.owner_group)['name']
@property
def users_group_name(self):
if self.users_group:
return di.get(KeycloakClient).group_get(self.users_group)['name']
return None
def to_dict(self):
data = {}
for column in self.__table__.columns:
if column.name.endswith('_quota_id'):
continue
for i in self._INLINE_CHILDS:
if column.name.startswith(f'{i}_'):
if i not in data:
data[i] = {}
data[i][column.name[len(i) + 1:]] = getattr(
self, column.name)
break
else:
data[column.name] = getattr(self, column.name)
for k in ['user_quota', 'total_quota']:
data[k] = getattr(self, k).to_dict() if getattr(self, k) else None
data['owner_group_name'] = self.owner_group_name
data['users_group_name'] = self.users_group_name
return data
@classmethod
def from_dict(cls, data):
data = copy.deepcopy(data)
for k in ['user_quota', 'total_quota']:
quota_data = data.pop(k, None)
if quota_data:
data[k] = Quota.from_dict(quota_data)
for i in cls._INLINE_CHILDS:
for k, v in data[i].items():
data[f'{i}_{k}'] = v
del data[i]
return super().from_dict(data)
def update_from_dict(self, data):
data = copy.deepcopy(data)
for k in ['user_quota', 'total_quota']:
if k in data:
if data[k]:
if getattr(self, k) is None:
setattr(self, k, Quota.from_dict(data[k]))
else:
setattr(self, k).update_from_dict(data[k])
else:
setattr(self, k, None)
del data[k]
for i in self._INLINE_CHILDS:
if i in data:
for k, v in data[i].items():
setattr(self, f'{i}_{k}', v)
del data[i]
super().update_from_dict(data)
def create_openstack_client(self, project=None):
"""
Create OpenStack SDK connection (client). Optional `project` argument
can be used to change project, default is project from the region
(:attr:`Region.project`).
Returns:
openstack.connection.Connection
"""
vault = di.get(Vault)
credentials = vault.read(self.openstack_credentials)
if not credentials:
raise RuntimeError(
f'Missing credentials in vault; {vault!r} {self.openstack_credentials}'
)
connection = openstack.connection.Connection(
auth=dict(
auth_url=self.openstack_url,
username=credentials['username'],
password=credentials['password'],
project_name=project or self.openstack_project,
domain_name=self.openstack_domain_name,
),
region_name="regionOne",
interface="public",
identity_api_version=3,
)
connection.authorize()
return connection
def get_user_project_name(self, user_id):
user_name = di.get(KeycloakClient).user_get(user_id)['username']
return f'ql_{user_name}'
def get_user_quota_usage(self, user_id):
query = (db.session.query(
db.func.coalesce(db.func.sum(ClusterHost.num_vcpus), 0),
db.func.coalesce(db.func.sum(ClusterHost.ram_mb), 0),
db.func.coalesce(db.func.sum(ClusterHost.num_volumes), 0),
db.func.coalesce(db.func.sum(ClusterHost.volumes_gb), 0),
).join(ClusterHost.cluster, ).where(
db.and_(
Cluster.region_id == self.id,
Cluster.user_id == user_id,
)))
result = query.first()
return dict(
zip(['num_vcpus', 'ram_mb', 'num_volumes', 'volumes_gb'], result))
def get_total_quota_usage(self):
query = (db.session.query(
db.func.coalesce(db.func.sum(ClusterHost.num_vcpus), 0),
db.func.coalesce(db.func.sum(ClusterHost.ram_mb), 0),
db.func.coalesce(db.func.sum(ClusterHost.num_volumes), 0),
db.func.coalesce(db.func.sum(ClusterHost.volumes_gb), 0),
).join(ClusterHost.cluster, ).where(Cluster.region_id == self.id, ))
result = query.first()
return dict(
zip(['num_vcpus', 'ram_mb', 'num_volumes', 'volumes_gb'], result))
def get_openstack_limits(self, project):
os_client = self.create_openstack_client(project)
return os_client.compute.get_limits()
def is_product_enabled(self, product_id):
"""Check if the product is configured and enabled in the region."""
region_product = self.products_relation.filter(
db.and_(
RegionProduct.product_id == product_id,
RegionProduct.enabled.is_(True),
), ).first()
return bool(region_product and region_product.product.enabled)