class Cluster(mb.SavannaBase):
"""Contains all info about cluster."""
__tablename__ = 'clusters'
__table_args__ = (sa.UniqueConstraint('name', 'tenant_id'), )
id = _id_column()
name = sa.Column(sa.String(80), nullable=False)
description = sa.Column(sa.Text)
tenant_id = sa.Column(sa.String(36))
plugin_name = sa.Column(sa.String(80), nullable=False)
hadoop_version = sa.Column(sa.String(80), nullable=False)
cluster_configs = sa.Column(st.JsonDictType())
default_image_id = sa.Column(sa.String(36))
anti_affinity = sa.Column(st.JsonListType())
private_key = sa.Column(sa.Text, default=crypto.generate_private_key())
user_keypair_id = sa.Column(sa.String(80))
status = sa.Column(sa.String(80))
status_description = sa.Column(sa.String(200))
info = sa.Column(st.JsonDictType())
node_groups = relationship('NodeGroup',
cascade="all,delete",
backref='cluster',
lazy='joined')
cluster_template_id = sa.Column(sa.String(36),
sa.ForeignKey('cluster_templates.id'))
cluster_template = relationship('ClusterTemplate',
backref="clusters",
lazy='joined')
def to_dict(self):
d = super(Cluster, self).to_dict()
d['node_groups'] = [ng.to_dict() for ng in self.node_groups]
return d
class TemplatesRelation(mb.SavannaBase):
"""NodeGroupTemplate - ClusterTemplate relationship.
In fact, it's a template of NodeGroup in Cluster.
"""
__tablename__ = 'templates_relations'
id = _id_column()
tenant_id = sa.Column(sa.String(36))
name = sa.Column(sa.String(80), nullable=False)
flavor_id = sa.Column(sa.String(36), nullable=False)
image_id = sa.Column(sa.String(36))
node_processes = sa.Column(st.JsonListType())
node_configs = sa.Column(st.JsonDictType())
volumes_per_node = sa.Column(sa.Integer)
volumes_size = sa.Column(sa.Integer)
volume_mount_prefix = sa.Column(sa.String(80))
count = sa.Column(sa.Integer, nullable=False)
cluster_template_id = sa.Column(sa.String(36),
sa.ForeignKey('cluster_templates.id'))
node_group_template_id = sa.Column(sa.String(36),
sa.ForeignKey(
'node_group_templates.id'))
node_group_template = relationship('NodeGroupTemplate',
backref="templates_relations",
lazy='joined')
floating_ip_pool = sa.Column(sa.String(36))
class ClusterTemplate(mb.SavannaBase):
"""Template for Cluster."""
__tablename__ = 'cluster_templates'
__table_args__ = (
sa.UniqueConstraint('name', 'tenant_id'),
)
id = _id_column()
name = sa.Column(sa.String(80), nullable=False)
description = sa.Column(sa.Text)
cluster_configs = sa.Column(st.JsonDictType())
default_image_id = sa.Column(sa.String(36))
anti_affinity = sa.Column(st.JsonListType())
tenant_id = sa.Column(sa.String(36))
neutron_management_network = sa.Column(sa.String(36))
plugin_name = sa.Column(sa.String(80), nullable=False)
hadoop_version = sa.Column(sa.String(80), nullable=False)
node_groups = relationship('TemplatesRelation', cascade="all,delete",
backref='cluster_template', lazy='joined')
def to_dict(self):
d = super(ClusterTemplate, self).to_dict()
d['node_groups'] = [tr.to_dict() for tr in
self.node_groups]
return d
class NodeGroup(mb.SavannaBase):
"""Specifies group of nodes within a cluster."""
__tablename__ = 'node_groups'
__table_args__ = (sa.UniqueConstraint('name', 'cluster_id'), )
id = _id_column()
name = sa.Column(sa.String(80), nullable=False)
flavor_id = sa.Column(sa.String(36), nullable=False)
image_id = sa.Column(sa.String(36))
node_processes = sa.Column(st.JsonListType())
node_configs = sa.Column(st.JsonDictType())
volumes_per_node = sa.Column(sa.Integer)
volumes_size = sa.Column(sa.Integer)
volume_mount_prefix = sa.Column(sa.String(80))
count = sa.Column(sa.Integer, nullable=False)
instances = relationship('Instance',
cascade="all,delete",
backref='node_group',
order_by="Instance.instance_name",
lazy='joined')
cluster_id = sa.Column(sa.String(36), sa.ForeignKey('clusters.id'))
node_group_template_id = sa.Column(
sa.String(36), sa.ForeignKey('node_group_templates.id'))
node_group_template = relationship('NodeGroupTemplate',
backref="node_groups",
lazy='joined')
def to_dict(self):
d = super(NodeGroup, self).to_dict()
d['instances'] = [i.to_dict() for i in self.instances]
return d
class JobExecution(mb.SavannaBase):
"""JobExecution - represent a job execution of specific cluster
"""
__tablename__ = 'job_executions'
id = _id_column()
tenant_id = sa.Column(sa.String(36))
job_id = sa.Column(sa.String(36), sa.ForeignKey('jobs.id'))
input_id = sa.Column(sa.String(36), sa.ForeignKey('data_sources.id'))
output_id = sa.Column(sa.String(36), sa.ForeignKey('data_sources.id'))
start_time = sa.Column(sa.DateTime())
end_time = sa.Column(sa.DateTime())
cluster_id = sa.Column(sa.String(36), sa.ForeignKey('clusters.id'))
info = sa.Column(st.JsonDictType())
progress = sa.Column(sa.Float)
oozie_job_id = sa.Column(sa.String(100))
return_code = sa.Column(sa.String(80))
job_configs = sa.Column(st.JsonDictType())
class JobBinary(mb.SavannaBase):
"""JobBinary - raw binary storage for executable jobs
"""
__tablename__ = 'job_binaries'
__table_args__ = (sa.UniqueConstraint('name', 'tenant_id'), )
id = _id_column()
tenant_id = sa.Column(sa.String(36))
name = sa.Column(sa.String(80), nullable=False)
description = sa.Column(sa.Text())
url = sa.Column(sa.String(256), nullable=False)
extra = sa.Column(st.JsonDictType())
class JobOrigin(mb.SavannaBase):
"""JobOrigin - description and location of a job binary
"""
__tablename__ = 'job_origins'
__table_args__ = (sa.UniqueConstraint('name', 'tenant_id'), )
id = _id_column()
tenant_id = sa.Column(sa.String(36))
name = sa.Column(sa.String(80), nullable=False)
description = sa.Column(sa.Text())
storage_type = sa.Column(sa.String(16))
url = sa.Column(sa.String())
credentials = sa.Column(st.JsonDictType())
class DataSource(mb.SavannaBase):
"""DataSource - represent a diffident types of data source,
e.g. Swift, Cassandra etc.
"""
__tablename__ = 'data_sources'
__table_args__ = (sa.UniqueConstraint('name', 'tenant_id'), )
id = _id_column()
tenant_id = sa.Column(sa.String(36))
name = sa.Column(sa.String(80), nullable=False)
description = sa.Column(sa.Text())
type = sa.Column(sa.String(80), nullable=False)
url = sa.Column(sa.String(256), nullable=False)
credentials = sa.Column(st.JsonDictType())
class NodeGroupTemplate(mb.SavannaBase):
"""Template for NodeGroup."""
__tablename__ = 'node_group_templates'
__table_args__ = (sa.UniqueConstraint('name', 'tenant_id'), )
id = _id_column()
name = sa.Column(sa.String(80), nullable=False)
description = sa.Column(sa.Text)
tenant_id = sa.Column(sa.String(36))
flavor_id = sa.Column(sa.String(36), nullable=False)
image_id = sa.Column(sa.String(36))
plugin_name = sa.Column(sa.String(80), nullable=False)
hadoop_version = sa.Column(sa.String(80), nullable=False)
node_processes = sa.Column(st.JsonListType())
node_configs = sa.Column(st.JsonDictType())
volumes_per_node = sa.Column(sa.Integer, nullable=False)
volumes_size = sa.Column(sa.Integer)
volume_mount_prefix = sa.Column(sa.String(80))
def upgrade():
op.create_table('job_binary_internal',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('name', sa.String(length=80), nullable=False),
sa.Column('data', sa.LargeBinary(), nullable=True),
sa.Column('datasize', sa.BIGINT(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', 'tenant_id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('node_group_templates',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=80), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('flavor_id',
sa.String(length=36),
nullable=False),
sa.Column('image_id', sa.String(length=36), nullable=True),
sa.Column('plugin_name',
sa.String(length=80),
nullable=False),
sa.Column('hadoop_version',
sa.String(length=80),
nullable=False),
sa.Column('node_processes',
st.JsonDictType(),
nullable=True),
sa.Column('node_configs', st.JsonDictType(),
nullable=True),
sa.Column('volumes_per_node', sa.Integer(),
nullable=False),
sa.Column('volumes_size', sa.Integer(), nullable=True),
sa.Column('volume_mount_prefix',
sa.String(length=80),
nullable=True),
sa.Column('floating_ip_pool',
sa.String(length=36),
nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', 'tenant_id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('data_sources',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('name', sa.String(length=80), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('type', sa.String(length=80), nullable=False),
sa.Column('url', sa.String(length=256), nullable=False),
sa.Column('credentials', st.JsonDictType(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', 'tenant_id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('cluster_templates',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=80), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('cluster_configs',
st.JsonDictType(),
nullable=True),
sa.Column('default_image_id',
sa.String(length=36),
nullable=True),
sa.Column('anti_affinity',
st.JsonDictType(),
nullable=True),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('neutron_management_network',
sa.String(length=36),
nullable=True),
sa.Column('plugin_name',
sa.String(length=80),
nullable=False),
sa.Column('hadoop_version',
sa.String(length=80),
nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', 'tenant_id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('job_binaries',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('name', sa.String(length=80), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('url', sa.String(length=256), nullable=False),
sa.Column('extra', st.JsonDictType(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', 'tenant_id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('jobs',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('name', sa.String(length=80), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('type', sa.String(length=80), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', 'tenant_id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('templates_relations',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('name', sa.String(length=80), nullable=False),
sa.Column('flavor_id',
sa.String(length=36),
nullable=False),
sa.Column('image_id', sa.String(length=36), nullable=True),
sa.Column('node_processes',
st.JsonDictType(),
nullable=True),
sa.Column('node_configs', st.JsonDictType(),
nullable=True),
sa.Column('volumes_per_node', sa.Integer(), nullable=True),
sa.Column('volumes_size', sa.Integer(), nullable=True),
sa.Column('volume_mount_prefix',
sa.String(length=80),
nullable=True),
sa.Column('count', sa.Integer(), nullable=False),
sa.Column('cluster_template_id',
sa.String(length=36),
nullable=True),
sa.Column('node_group_template_id',
sa.String(length=36),
nullable=True),
sa.Column('floating_ip_pool',
sa.String(length=36),
nullable=True),
sa.ForeignKeyConstraint(
['cluster_template_id'],
['cluster_templates.id'],
),
sa.ForeignKeyConstraint(
['node_group_template_id'],
['node_group_templates.id'],
),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('mains_association',
sa.Column('Job_id', sa.String(length=36), nullable=True),
sa.Column('JobBinary_id',
sa.String(length=36),
nullable=True),
sa.ForeignKeyConstraint(
['JobBinary_id'],
['job_binaries.id'],
),
sa.ForeignKeyConstraint(
['Job_id'],
['jobs.id'],
),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('libs_association',
sa.Column('Job_id', sa.String(length=36), nullable=True),
sa.Column('JobBinary_id',
sa.String(length=36),
nullable=True),
sa.ForeignKeyConstraint(
['JobBinary_id'],
['job_binaries.id'],
),
sa.ForeignKeyConstraint(
['Job_id'],
['jobs.id'],
),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('clusters',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=80), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('trust_id', sa.String(length=36), nullable=True),
sa.Column('is_transient', sa.Boolean(), nullable=True),
sa.Column('plugin_name',
sa.String(length=80),
nullable=False),
sa.Column('hadoop_version',
sa.String(length=80),
nullable=False),
sa.Column('cluster_configs',
st.JsonDictType(),
nullable=True),
sa.Column('default_image_id',
sa.String(length=36),
nullable=True),
sa.Column('neutron_management_network',
sa.String(length=36),
nullable=True),
sa.Column('anti_affinity',
st.JsonDictType(),
nullable=True),
sa.Column('management_private_key',
sa.Text(),
nullable=False),
sa.Column('management_public_key',
sa.Text(),
nullable=False),
sa.Column('user_keypair_id',
sa.String(length=80),
nullable=True),
sa.Column('status', sa.String(length=80), nullable=True),
sa.Column('status_description',
sa.String(length=200),
nullable=True),
sa.Column('info', st.JsonDictType(), nullable=True),
sa.Column('extra', st.JsonDictType(), nullable=True),
sa.Column('cluster_template_id',
sa.String(length=36),
nullable=True),
sa.ForeignKeyConstraint(
['cluster_template_id'],
['cluster_templates.id'],
),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', 'tenant_id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('node_groups',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=80), nullable=False),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('flavor_id',
sa.String(length=36),
nullable=False),
sa.Column('image_id', sa.String(length=36), nullable=True),
sa.Column('image_username',
sa.String(length=36),
nullable=True),
sa.Column('node_processes',
st.JsonDictType(),
nullable=True),
sa.Column('node_configs', st.JsonDictType(),
nullable=True),
sa.Column('volumes_per_node', sa.Integer(), nullable=True),
sa.Column('volumes_size', sa.Integer(), nullable=True),
sa.Column('volume_mount_prefix',
sa.String(length=80),
nullable=True),
sa.Column('count', sa.Integer(), nullable=False),
sa.Column('cluster_id',
sa.String(length=36),
nullable=True),
sa.Column('node_group_template_id',
sa.String(length=36),
nullable=True),
sa.Column('floating_ip_pool',
sa.String(length=36),
nullable=True),
sa.ForeignKeyConstraint(
['cluster_id'],
['clusters.id'],
),
sa.ForeignKeyConstraint(
['node_group_template_id'],
['node_group_templates.id'],
),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', 'cluster_id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('job_executions',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('job_id', sa.String(length=36), nullable=True),
sa.Column('input_id', sa.String(length=36), nullable=True),
sa.Column('output_id', sa.String(length=36),
nullable=True),
sa.Column('start_time', sa.DateTime(), nullable=True),
sa.Column('end_time', sa.DateTime(), nullable=True),
sa.Column('cluster_id',
sa.String(length=36),
nullable=True),
sa.Column('info', st.JsonDictType(), nullable=True),
sa.Column('progress', sa.Float(), nullable=True),
sa.Column('oozie_job_id',
sa.String(length=100),
nullable=True),
sa.Column('return_code',
sa.String(length=80),
nullable=True),
sa.Column('job_configs', st.JsonDictType(), nullable=True),
sa.Column('main_class', sa.Text(), nullable=True),
sa.Column('java_opts', sa.Text(), nullable=True),
sa.ForeignKeyConstraint(
['cluster_id'],
['clusters.id'],
),
sa.ForeignKeyConstraint(
['input_id'],
['data_sources.id'],
),
sa.ForeignKeyConstraint(
['job_id'],
['jobs.id'],
),
sa.ForeignKeyConstraint(
['output_id'],
['data_sources.id'],
),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table('instances',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('tenant_id', sa.String(length=36),
nullable=True),
sa.Column('node_group_id',
sa.String(length=36),
nullable=True),
sa.Column('instance_id',
sa.String(length=36),
nullable=True),
sa.Column('instance_name',
sa.String(length=80),
nullable=False),
sa.Column('internal_ip',
sa.String(length=15),
nullable=True),
sa.Column('management_ip',
sa.String(length=15),
nullable=True),
sa.Column('volumes', st.JsonDictType(), nullable=True),
sa.ForeignKeyConstraint(
['node_group_id'],
['node_groups.id'],
),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('instance_id', 'node_group_id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)