class Cluster(mb.SavannaBase): """Contains all info about cluster.""" __tablename__ = 'clusters' __table_args__ = (sa.UniqueConstraint('name', 'tenant_id'), ) id = _id_column() name = sa.Column(sa.String(80), nullable=False) description = sa.Column(sa.Text) tenant_id = sa.Column(sa.String(36)) plugin_name = sa.Column(sa.String(80), nullable=False) hadoop_version = sa.Column(sa.String(80), nullable=False) cluster_configs = sa.Column(st.JsonDictType()) default_image_id = sa.Column(sa.String(36)) anti_affinity = sa.Column(st.JsonListType()) private_key = sa.Column(sa.Text, default=crypto.generate_private_key()) user_keypair_id = sa.Column(sa.String(80)) status = sa.Column(sa.String(80)) status_description = sa.Column(sa.String(200)) info = sa.Column(st.JsonDictType()) node_groups = relationship('NodeGroup', cascade="all,delete", backref='cluster', lazy='joined') cluster_template_id = sa.Column(sa.String(36), sa.ForeignKey('cluster_templates.id')) cluster_template = relationship('ClusterTemplate', backref="clusters", lazy='joined') def to_dict(self): d = super(Cluster, self).to_dict() d['node_groups'] = [ng.to_dict() for ng in self.node_groups] return d
class TemplatesRelation(mb.SavannaBase): """NodeGroupTemplate - ClusterTemplate relationship. In fact, it's a template of NodeGroup in Cluster. """ __tablename__ = 'templates_relations' id = _id_column() tenant_id = sa.Column(sa.String(36)) name = sa.Column(sa.String(80), nullable=False) flavor_id = sa.Column(sa.String(36), nullable=False) image_id = sa.Column(sa.String(36)) node_processes = sa.Column(st.JsonListType()) node_configs = sa.Column(st.JsonDictType()) volumes_per_node = sa.Column(sa.Integer) volumes_size = sa.Column(sa.Integer) volume_mount_prefix = sa.Column(sa.String(80)) count = sa.Column(sa.Integer, nullable=False) cluster_template_id = sa.Column(sa.String(36), sa.ForeignKey('cluster_templates.id')) node_group_template_id = sa.Column(sa.String(36), sa.ForeignKey( 'node_group_templates.id')) node_group_template = relationship('NodeGroupTemplate', backref="templates_relations", lazy='joined') floating_ip_pool = sa.Column(sa.String(36))
class ClusterTemplate(mb.SavannaBase): """Template for Cluster.""" __tablename__ = 'cluster_templates' __table_args__ = ( sa.UniqueConstraint('name', 'tenant_id'), ) id = _id_column() name = sa.Column(sa.String(80), nullable=False) description = sa.Column(sa.Text) cluster_configs = sa.Column(st.JsonDictType()) default_image_id = sa.Column(sa.String(36)) anti_affinity = sa.Column(st.JsonListType()) tenant_id = sa.Column(sa.String(36)) neutron_management_network = sa.Column(sa.String(36)) plugin_name = sa.Column(sa.String(80), nullable=False) hadoop_version = sa.Column(sa.String(80), nullable=False) node_groups = relationship('TemplatesRelation', cascade="all,delete", backref='cluster_template', lazy='joined') def to_dict(self): d = super(ClusterTemplate, self).to_dict() d['node_groups'] = [tr.to_dict() for tr in self.node_groups] return d
class NodeGroup(mb.SavannaBase): """Specifies group of nodes within a cluster.""" __tablename__ = 'node_groups' __table_args__ = (sa.UniqueConstraint('name', 'cluster_id'), ) id = _id_column() name = sa.Column(sa.String(80), nullable=False) flavor_id = sa.Column(sa.String(36), nullable=False) image_id = sa.Column(sa.String(36)) node_processes = sa.Column(st.JsonListType()) node_configs = sa.Column(st.JsonDictType()) volumes_per_node = sa.Column(sa.Integer) volumes_size = sa.Column(sa.Integer) volume_mount_prefix = sa.Column(sa.String(80)) count = sa.Column(sa.Integer, nullable=False) instances = relationship('Instance', cascade="all,delete", backref='node_group', order_by="Instance.instance_name", lazy='joined') cluster_id = sa.Column(sa.String(36), sa.ForeignKey('clusters.id')) node_group_template_id = sa.Column( sa.String(36), sa.ForeignKey('node_group_templates.id')) node_group_template = relationship('NodeGroupTemplate', backref="node_groups", lazy='joined') def to_dict(self): d = super(NodeGroup, self).to_dict() d['instances'] = [i.to_dict() for i in self.instances] return d
class JobExecution(mb.SavannaBase): """JobExecution - represent a job execution of specific cluster """ __tablename__ = 'job_executions' id = _id_column() tenant_id = sa.Column(sa.String(36)) job_id = sa.Column(sa.String(36), sa.ForeignKey('jobs.id')) input_id = sa.Column(sa.String(36), sa.ForeignKey('data_sources.id')) output_id = sa.Column(sa.String(36), sa.ForeignKey('data_sources.id')) start_time = sa.Column(sa.DateTime()) end_time = sa.Column(sa.DateTime()) cluster_id = sa.Column(sa.String(36), sa.ForeignKey('clusters.id')) info = sa.Column(st.JsonDictType()) progress = sa.Column(sa.Float) oozie_job_id = sa.Column(sa.String(100)) return_code = sa.Column(sa.String(80)) job_configs = sa.Column(st.JsonDictType())
class JobBinary(mb.SavannaBase): """JobBinary - raw binary storage for executable jobs """ __tablename__ = 'job_binaries' __table_args__ = (sa.UniqueConstraint('name', 'tenant_id'), ) id = _id_column() tenant_id = sa.Column(sa.String(36)) name = sa.Column(sa.String(80), nullable=False) description = sa.Column(sa.Text()) url = sa.Column(sa.String(256), nullable=False) extra = sa.Column(st.JsonDictType())
class JobOrigin(mb.SavannaBase): """JobOrigin - description and location of a job binary """ __tablename__ = 'job_origins' __table_args__ = (sa.UniqueConstraint('name', 'tenant_id'), ) id = _id_column() tenant_id = sa.Column(sa.String(36)) name = sa.Column(sa.String(80), nullable=False) description = sa.Column(sa.Text()) storage_type = sa.Column(sa.String(16)) url = sa.Column(sa.String()) credentials = sa.Column(st.JsonDictType())
class DataSource(mb.SavannaBase): """DataSource - represent a diffident types of data source, e.g. Swift, Cassandra etc. """ __tablename__ = 'data_sources' __table_args__ = (sa.UniqueConstraint('name', 'tenant_id'), ) id = _id_column() tenant_id = sa.Column(sa.String(36)) name = sa.Column(sa.String(80), nullable=False) description = sa.Column(sa.Text()) type = sa.Column(sa.String(80), nullable=False) url = sa.Column(sa.String(256), nullable=False) credentials = sa.Column(st.JsonDictType())
class NodeGroupTemplate(mb.SavannaBase): """Template for NodeGroup.""" __tablename__ = 'node_group_templates' __table_args__ = (sa.UniqueConstraint('name', 'tenant_id'), ) id = _id_column() name = sa.Column(sa.String(80), nullable=False) description = sa.Column(sa.Text) tenant_id = sa.Column(sa.String(36)) flavor_id = sa.Column(sa.String(36), nullable=False) image_id = sa.Column(sa.String(36)) plugin_name = sa.Column(sa.String(80), nullable=False) hadoop_version = sa.Column(sa.String(80), nullable=False) node_processes = sa.Column(st.JsonListType()) node_configs = sa.Column(st.JsonDictType()) volumes_per_node = sa.Column(sa.Integer, nullable=False) volumes_size = sa.Column(sa.Integer) volume_mount_prefix = sa.Column(sa.String(80))
def upgrade(): op.create_table('job_binary_internal', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('name', sa.String(length=80), nullable=False), sa.Column('data', sa.LargeBinary(), nullable=True), sa.Column('datasize', sa.BIGINT(), nullable=True), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('name', 'tenant_id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('node_group_templates', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('name', sa.String(length=80), nullable=False), sa.Column('description', sa.Text(), nullable=True), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('flavor_id', sa.String(length=36), nullable=False), sa.Column('image_id', sa.String(length=36), nullable=True), sa.Column('plugin_name', sa.String(length=80), nullable=False), sa.Column('hadoop_version', sa.String(length=80), nullable=False), sa.Column('node_processes', st.JsonDictType(), nullable=True), sa.Column('node_configs', st.JsonDictType(), nullable=True), sa.Column('volumes_per_node', sa.Integer(), nullable=False), sa.Column('volumes_size', sa.Integer(), nullable=True), sa.Column('volume_mount_prefix', sa.String(length=80), nullable=True), sa.Column('floating_ip_pool', sa.String(length=36), nullable=True), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('name', 'tenant_id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('data_sources', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('name', sa.String(length=80), nullable=False), sa.Column('description', sa.Text(), nullable=True), sa.Column('type', sa.String(length=80), nullable=False), sa.Column('url', sa.String(length=256), nullable=False), sa.Column('credentials', st.JsonDictType(), nullable=True), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('name', 'tenant_id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('cluster_templates', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('name', sa.String(length=80), nullable=False), sa.Column('description', sa.Text(), nullable=True), sa.Column('cluster_configs', st.JsonDictType(), nullable=True), sa.Column('default_image_id', sa.String(length=36), nullable=True), sa.Column('anti_affinity', st.JsonDictType(), nullable=True), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('neutron_management_network', sa.String(length=36), nullable=True), sa.Column('plugin_name', sa.String(length=80), nullable=False), sa.Column('hadoop_version', sa.String(length=80), nullable=False), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('name', 'tenant_id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('job_binaries', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('name', sa.String(length=80), nullable=False), sa.Column('description', sa.Text(), nullable=True), sa.Column('url', sa.String(length=256), nullable=False), sa.Column('extra', st.JsonDictType(), nullable=True), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('name', 'tenant_id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('jobs', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('name', sa.String(length=80), nullable=False), sa.Column('description', sa.Text(), nullable=True), sa.Column('type', sa.String(length=80), nullable=False), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('name', 'tenant_id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('templates_relations', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('name', sa.String(length=80), nullable=False), sa.Column('flavor_id', sa.String(length=36), nullable=False), sa.Column('image_id', sa.String(length=36), nullable=True), sa.Column('node_processes', st.JsonDictType(), nullable=True), sa.Column('node_configs', st.JsonDictType(), nullable=True), sa.Column('volumes_per_node', sa.Integer(), nullable=True), sa.Column('volumes_size', sa.Integer(), nullable=True), sa.Column('volume_mount_prefix', sa.String(length=80), nullable=True), sa.Column('count', sa.Integer(), nullable=False), sa.Column('cluster_template_id', sa.String(length=36), nullable=True), sa.Column('node_group_template_id', sa.String(length=36), nullable=True), sa.Column('floating_ip_pool', sa.String(length=36), nullable=True), sa.ForeignKeyConstraint( ['cluster_template_id'], ['cluster_templates.id'], ), sa.ForeignKeyConstraint( ['node_group_template_id'], ['node_group_templates.id'], ), sa.PrimaryKeyConstraint('id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('mains_association', sa.Column('Job_id', sa.String(length=36), nullable=True), sa.Column('JobBinary_id', sa.String(length=36), nullable=True), sa.ForeignKeyConstraint( ['JobBinary_id'], ['job_binaries.id'], ), sa.ForeignKeyConstraint( ['Job_id'], ['jobs.id'], ), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('libs_association', sa.Column('Job_id', sa.String(length=36), nullable=True), sa.Column('JobBinary_id', sa.String(length=36), nullable=True), sa.ForeignKeyConstraint( ['JobBinary_id'], ['job_binaries.id'], ), sa.ForeignKeyConstraint( ['Job_id'], ['jobs.id'], ), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('clusters', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('name', sa.String(length=80), nullable=False), sa.Column('description', sa.Text(), nullable=True), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('trust_id', sa.String(length=36), nullable=True), sa.Column('is_transient', sa.Boolean(), nullable=True), sa.Column('plugin_name', sa.String(length=80), nullable=False), sa.Column('hadoop_version', sa.String(length=80), nullable=False), sa.Column('cluster_configs', st.JsonDictType(), nullable=True), sa.Column('default_image_id', sa.String(length=36), nullable=True), sa.Column('neutron_management_network', sa.String(length=36), nullable=True), sa.Column('anti_affinity', st.JsonDictType(), nullable=True), sa.Column('management_private_key', sa.Text(), nullable=False), sa.Column('management_public_key', sa.Text(), nullable=False), sa.Column('user_keypair_id', sa.String(length=80), nullable=True), sa.Column('status', sa.String(length=80), nullable=True), sa.Column('status_description', sa.String(length=200), nullable=True), sa.Column('info', st.JsonDictType(), nullable=True), sa.Column('extra', st.JsonDictType(), nullable=True), sa.Column('cluster_template_id', sa.String(length=36), nullable=True), sa.ForeignKeyConstraint( ['cluster_template_id'], ['cluster_templates.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('name', 'tenant_id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('node_groups', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('name', sa.String(length=80), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('flavor_id', sa.String(length=36), nullable=False), sa.Column('image_id', sa.String(length=36), nullable=True), sa.Column('image_username', sa.String(length=36), nullable=True), sa.Column('node_processes', st.JsonDictType(), nullable=True), sa.Column('node_configs', st.JsonDictType(), nullable=True), sa.Column('volumes_per_node', sa.Integer(), nullable=True), sa.Column('volumes_size', sa.Integer(), nullable=True), sa.Column('volume_mount_prefix', sa.String(length=80), nullable=True), sa.Column('count', sa.Integer(), nullable=False), sa.Column('cluster_id', sa.String(length=36), nullable=True), sa.Column('node_group_template_id', sa.String(length=36), nullable=True), sa.Column('floating_ip_pool', sa.String(length=36), nullable=True), sa.ForeignKeyConstraint( ['cluster_id'], ['clusters.id'], ), sa.ForeignKeyConstraint( ['node_group_template_id'], ['node_group_templates.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('name', 'cluster_id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('job_executions', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('job_id', sa.String(length=36), nullable=True), sa.Column('input_id', sa.String(length=36), nullable=True), sa.Column('output_id', sa.String(length=36), nullable=True), sa.Column('start_time', sa.DateTime(), nullable=True), sa.Column('end_time', sa.DateTime(), nullable=True), sa.Column('cluster_id', sa.String(length=36), nullable=True), sa.Column('info', st.JsonDictType(), nullable=True), sa.Column('progress', sa.Float(), nullable=True), sa.Column('oozie_job_id', sa.String(length=100), nullable=True), sa.Column('return_code', sa.String(length=80), nullable=True), sa.Column('job_configs', st.JsonDictType(), nullable=True), sa.Column('main_class', sa.Text(), nullable=True), sa.Column('java_opts', sa.Text(), nullable=True), sa.ForeignKeyConstraint( ['cluster_id'], ['clusters.id'], ), sa.ForeignKeyConstraint( ['input_id'], ['data_sources.id'], ), sa.ForeignKeyConstraint( ['job_id'], ['jobs.id'], ), sa.ForeignKeyConstraint( ['output_id'], ['data_sources.id'], ), sa.PrimaryKeyConstraint('id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET) op.create_table('instances', sa.Column('created_at', sa.DateTime(), nullable=True), sa.Column('updated_at', sa.DateTime(), nullable=True), sa.Column('id', sa.String(length=36), nullable=False), sa.Column('tenant_id', sa.String(length=36), nullable=True), sa.Column('node_group_id', sa.String(length=36), nullable=True), sa.Column('instance_id', sa.String(length=36), nullable=True), sa.Column('instance_name', sa.String(length=80), nullable=False), sa.Column('internal_ip', sa.String(length=15), nullable=True), sa.Column('management_ip', sa.String(length=15), nullable=True), sa.Column('volumes', st.JsonDictType(), nullable=True), sa.ForeignKeyConstraint( ['node_group_id'], ['node_groups.id'], ), sa.PrimaryKeyConstraint('id'), sa.UniqueConstraint('instance_id', 'node_group_id'), mysql_engine=MYSQL_ENGINE, mysql_charset=MYSQL_CHARSET)