class PermissionAssignment(morpfw.sql.Base):
__tablename__ = "morpcc_permissionassignment"
model = sa.Column(sa.String(length=256))
permission = sa.Column(sa.String(length=256))
is_creator = sa.Column(sa.Boolean())
users = sa.Column(sajson.JSONField())
groups = sa.Column(sajson.JSONField())
roles = sa.Column(sajson.JSONField())
rule = sa.Column(sa.String(length=24))
enabled = sa.Column(sa.Boolean())
def dataclass_field_to_sqla_col(prop: dataclasses.Field) -> sqlalchemy.Column:
t = dataclass_get_type(prop)
if t["type"] == date:
params = sqlalchemy_params(prop, typ=sqlalchemy.Date())
return sqlalchemy.Column(**params)
if t["type"] == datetime:
params = sqlalchemy_params(prop, typ=sqlalchemy.DateTime(timezone=True))
return sqlalchemy.Column(**params)
if t["type"] == str:
str_format = t["metadata"].get("format", None)
if str_format and "/" in str_format:
str_format = str_format.split("/")[0]
if str_format == "text":
params = sqlalchemy_params(prop, typ=sqlalchemy.Text())
elif str_format == "uuid":
params = sqlalchemy_params(prop, typ=sautils.UUIDType())
elif str_format == "fulltextindex":
params = sqlalchemy_params(prop, typ=sautils.TSVectorType)
else:
str_len = prop.metadata.get("length", 256)
params = sqlalchemy_params(prop, typ=sqlalchemy.String(str_len))
return sqlalchemy.Column(**params)
if t["type"] == int:
if t["metadata"].get("format", None) == "bigint":
params = sqlalchemy_params(prop, typ=sqlalchemy.BigInteger())
else:
params = sqlalchemy_params(prop, typ=sqlalchemy.Integer())
return sqlalchemy.Column(**params)
if t["type"] == float:
if t["metadata"].get("format", None) == "numeric":
params = sqlalchemy_params(prop, typ=sqlalchemy.Numeric())
else:
params = sqlalchemy_params(prop, typ=sqlalchemy.Float())
return sqlalchemy.Column(**params)
if t["type"] == bool:
params = sqlalchemy_params(prop, typ=sqlalchemy.Boolean())
return sqlalchemy.Column(**params)
if is_dataclass_field(prop):
raise NotImplementedError("Sub schema is not supported")
if t["type"] == dict:
params = sqlalchemy_params(prop, typ=sajson.JSONField())
return sqlalchemy.Column(**params)
if t["type"] == list:
params = sqlalchemy_params(prop, typ=sajson.JSONField())
return sqlalchemy.Column(**params)
raise KeyError(prop)
class Chore(Base):
__tablename__ = "chore"
chore_id = sqlalchemy.Column(sqlalchemy.Integer,
primary_key=True,
autoincrement=True)
person_id = sqlalchemy.Column(sqlalchemy.Integer,
sqlalchemy.ForeignKey("person.person_id"),
nullable=False)
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
status = sqlalchemy.Column(sqlalchemy.Enum("started", "ended"))
created = sqlalchemy.Column(sqlalchemy.Integer)
updated = sqlalchemy.Column(sqlalchemy.Integer)
data = sqlalchemy.Column(sqlalchemy.ext.mutable.MutableDict.as_mutable(
sqlalchemy_jsonfield.JSONField(enforce_string=True,
enforce_unicode=False)),
nullable=False)
person = sqlalchemy.orm.relationship("Person")
sqlalchemy.schema.UniqueConstraint('name',
'person_id',
'created',
name='label')
def __repr__(self):
return "<Chore(name='%s',person='%s',created=%s)>" % (
self.name, self.person.name, self.created)
class Offer(Base):
__tablename__ = "offers"
offer_id = orm.Column(
orm.String(64),
primary_key=True,
autoincrement=False,
)
project_id = orm.Column(orm.String(64), nullable=False)
status = orm.Column(orm.String(15),
nullable=False,
default=statuses.AVAILABLE)
resource_id = orm.Column(orm.String(64), nullable=False)
resource_type = orm.Column(orm.String(64),
nullable=False,
default=resource_types.IRONIC_NODE)
start_time = orm.Column(orm.DateTime(timezone=True), nullable=False)
end_time = orm.Column(orm.DateTime(timezone=True), nullable=True)
config = orm.Column(
sqlalchemy_jsonfield.JSONField(enforce_string=True,
enforce_unicode=False),
nullable=False,
)
cost = orm.Column(orm.Float, nullable=False)
offer_contract_relationships = orm.relationship(
'OfferContractRelationship', lazy='dynamic')
@orm.validates('cost')
def validate_cost(self, key, value):
if value < 0:
raise ValueError('Cost must be >= 0')
return value
class invitation(Base):
__tablename__ = 'rsvp'
id = Column(
String,
primary_key=True,
)
guest = Column(sqlalchemy_jsonfield.JSONField(enforce_string=False,
enforce_unicode=False),
nullable=False)
email = Column(String, nullable=True)
comment = Column(String, nullable=True)
phone = Column(String, nullable=True)
rsvped = Column(Boolean)
table = Column(Boolean)
date = Column(DateTime(), nullable=True)
@property
def serialize(self):
"""Return object data in easily serializeable format"""
return {
'id': self.id,
'guest': self.guest,
'email': self.email,
'comment': self.comment,
'phone': self.phone,
'rsvped': self.rsvped,
'table': self.table,
'date': dump_datetime(self.date)
}
class Offer(Base):
__tablename__ = "offers"
marketplace_offer_id = orm.Column(
orm.String(64),
primary_key=True,
autoincrement=False,
)
provider_offer_id = orm.Column(
orm.String(64),
nullable=False,
unique=True,
)
project_id = orm.Column(orm.String(64), nullable=False)
status = orm.Column(orm.String(15), nullable=False, default="available")
server_id = orm.Column(orm.String(64), nullable=False, unique=True)
start_time = orm.Column(orm.DateTime(timezone=True), nullable=False)
end_time = orm.Column(orm.DateTime(timezone=True), nullable=True)
server_config = orm.Column(
sqlalchemy_jsonfield.JSONField(enforce_string=True,
enforce_unicode=False),
nullable=False,
)
cost = orm.Column(orm.Float, nullable=False)
offer_contract_relationships = orm.relationship(
'OfferContractRelationship', lazy='dynamic')
def _create_dataset_table():
op.create_table(
'dataset',
sa.Column('id', Integer, primary_key=True, autoincrement=True),
sa.Column(
'uri',
String(length=3000).with_variant(
String(
length=3000,
# latin1 allows for more indexed length in mysql
# and this field should only be ascii chars
collation='latin1_general_cs',
),
'mysql',
),
nullable=False,
),
sa.Column('extra',
sqlalchemy_jsonfield.JSONField(json=json),
nullable=False,
default={}),
sa.Column('created_at', TIMESTAMP, nullable=False),
sa.Column('updated_at', TIMESTAMP, nullable=False),
sqlite_autoincrement=True, # ensures PK values not reused
)
op.create_index('idx_uri_unique', 'dataset', ['uri'], unique=True)
class Result(db.Model):
id = db.Column(db.Integer, primary_key=True)
file_id = db.Column(db.String(50))
file_data = db.Column( sqlalchemy_jsonfield.JSONField( enforce_string=True, enforce_unicode=False, json=ujson, ), )
def __repr__(self):
return "Result('{id}','{file_id}','{file_data}')".format(id= self.id, file_id=self.file_id, file_data=self.file_data)
class GenericField(db.Model):
"""
Holds generic fields that can be added to different risk types.
"""
__tablename__ = 'generic_fields'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.Text, unique=True)
description = db.Column(db.Text)
type = db.Column(db.Enum(FieldType))
options = db.Column(sqlalchemy_jsonfield.JSONField())
def __init__(self, name, type, description='', options=None):
self.name = name
self.description = description
self.type = type
self.options = options or {}
def __repr__(self):
return '<Field {}: {}>'.format(self.id, self.name)
@validates('options')
def validate_address(self, key, options):
return validate_field_options(options, self.type)
def to_dict(self):
return {
'id': self.id,
'name': self.name,
'description': self.description,
'type': self.type.value,
'options': self.options,
}
class ArtItem(Base):
__versioned__ = {}
__tablename__ = 'art_item'
id = Column(BigInteger, primary_key = True, index = True)
state = Column(dlstate_enum, default='new', index=True, nullable=False)
errno = Column(Integer, default='0')
artist_id = Column(BigInteger, ForeignKey('scrape_targets.id'), nullable=False)
release_meta = Column(Text, nullable = False, index=True)
fetchtime = Column(DateTime, default=datetime.datetime.min)
addtime = Column(DateTime, default=datetime.datetime.utcnow)
title = Column(Text)
content = Column(Text)
content_structured = Column(sqlalchemy_jsonfield.JSONField())
artist = relationship("ScrapeTargets")
files = relationship("ArtFile")
tags = relationship("ArtTags")
__table_args__ = (
UniqueConstraint('artist_id', 'release_meta'),
)
class Hospital(db.Model):
hospital_id = db.Column(db.Integer, primary_key=True)
hospital_name = db.Column(db.String(255))
hospital_address = db.Column(db.String(255))
hospital_phno = db.Column(db.String(50))
hospital_image = db.Column(db.String(255))
hospital_location = db.Column(sqlalchemy_jsonfield.JSONField())
class Bid(Base):
__tablename__ = 'bids'
bid_id = orm.Column(
orm.String(64),
primary_key=True,
autoincrement=False,
)
project_id = orm.Column(orm.String(64), nullable=False)
quantity = orm.Column(orm.Integer, nullable=False)
start_time = orm.Column(orm.DateTime(timezone=True), nullable=False)
end_time = orm.Column(orm.DateTime(timezone=True), nullable=False)
duration = orm.Column(orm.Integer, nullable=False)
status = orm.Column(orm.String(15),
nullable=False,
default=statuses.AVAILABLE)
config_query = orm.Column(sqlalchemy_jsonfield.JSONField(
enforce_string=True, enforce_unicode=False),
nullable=False)
cost = orm.Column(orm.Float, nullable=False)
contracts = orm.relationship('Contract', lazy='dynamic')
@orm.validates('cost')
def validate_cost(self, key, value):
if value < 0:
raise ValueError('Cost must be >= 0')
return value
class LobbyOptions(Base):
__tablename__ = 'lobby_options'
id = Column(Integer, primary_key=True)
name = Column(String(127), unique=True)
options = Column(sqlalchemy_jsonfield.JSONField())
@classmethod
def save_options(cls, name: str, options: t.Mapping[str, t.Any]) -> None:
try:
EDB.Session.add(cls(name=name, options=options))
EDB.Session.commit()
except IntegrityError:
EDB.Session.rollback()
EDB.Session.execute(
update(cls).where(cls.name == name).values(options=options))
EDB.Session.commit()
@classmethod
def get_options_for_name(cls,
name: str) -> t.Optional[t.Mapping[str, t.Any]]:
instance = EDB.Session.query(
cls.options).filter(cls.name == name).first()
if instance is None:
return instance
return instance.options
class ExampleTable(Base):
__tablename__ = table_name
id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
row_name = sqlalchemy.Column(sqlalchemy.Unicode(64), unique=True)
json_record = sqlalchemy.Column(
sqlalchemy_jsonfield.JSONField(enforce_string=True),
nullable=False # MariaDB does not support JSON for now
)
class ExampleTable(Base):
__tablename__ = table_name
id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
row_name = sqlalchemy.Column(
sqlalchemy.Unicode(64),
unique=True,
)
json_record = sqlalchemy.Column(sqlalchemy_jsonfield.JSONField(),
nullable=False)
class Mapping(db.Model):
id = db.Column(db.Integer, primary_key=True)
mapping = db.Column(sqlalchemy_jsonfield.JSONField(enforce_string=True,
enforce_unicode=False),
nullable=False)
def __init__(self, id, mapping):
self.id = id
self.mapping = mapping
class Process(morpfw.sql.Base):
__tablename__ = "morpcc_process"
signal = sa.Column(sa.String(length=1024), index=True)
task_id = sa.Column(morpfw.sql.GUID(), index=True)
start = sa.Column(sa.DateTime(timezone=True), index=True)
end = sa.Column(sa.DateTime(timezone=True), index=True)
params = sa.Column(sajson.JSONField())
traceback = sa.Column(sa.Text())
def tables(for_downgrade=False):
import sqlalchemy_jsonfield
global task_instance, rendered_task_instance_fields, dag_run
metadata = sa.MetaData()
task_instance = sa.Table(
'task_instance',
metadata,
sa.Column('task_id', StringID()),
sa.Column('dag_id', StringID()),
sa.Column('run_id', StringID()),
sa.Column('execution_date', TIMESTAMP),
)
rendered_task_instance_fields = sa.Table(
'rendered_task_instance_fields',
metadata,
sa.Column('dag_id', StringID()),
sa.Column('task_id', StringID()),
sa.Column('run_id', StringID()),
sa.Column('execution_date', TIMESTAMP),
sa.Column('rendered_fields', sqlalchemy_jsonfield.JSONField(), nullable=False),
sa.Column('k8s_pod_yaml', sqlalchemy_jsonfield.JSONField(), nullable=True),
)
if for_downgrade:
rendered_task_instance_fields.append_column(
sa.Column('map_index', sa.Integer(), server_default='-1'),
)
rendered_task_instance_fields.append_constraint(
ForeignKeyConstraint(
['dag_id', 'run_id'],
["dag_run.dag_id", "dag_run.run_id"],
name='rtif_dag_run_fkey',
ondelete="CASCADE",
),
)
dag_run = sa.Table(
'dag_run',
metadata,
sa.Column('dag_id', StringID()),
sa.Column('run_id', StringID()),
sa.Column('execution_date', TIMESTAMP),
)
class ActivityLog(morpfw.sql.Base):
__tablename__ = "morpcc_activitylog"
userid = sa.Column(morpfw.sql.GUID())
resource_uuid = sa.Column(morpfw.sql.GUID())
metalink_type = sa.Column(sa.String(length=128))
metalink = sa.Column(sajson.JSONField())
view_name = sa.Column(sa.String(length=128))
source_ip = sa.Column(sa.String(length=64))
activity = sa.Column(sa.Text())
request_url = sa.Column(sa.String(length=2000))
request_method = sa.Column(sa.String(length=12))
class Offers(Base):
__tablename__ = 'offers'
offer_id = Column(String(64), primary_key=True, autoincrement=False)
project_id = Column(String(64), nullable=False)
status = Column(String(16), nullable=False, default=statuses.AVAILABLE)
resource_id = Column(String(64), nullable=False)
# resource_type = Column(String(100), nullable=False)
start_time = Column(DateTime(timezone=True), nullable=False)
end_time = Column(DateTime(timezone=True), nullable=False)
config = Column(sqlalchemy_jsonfield.JSONField(
enforce_string=True,
enforce_unicode=False), nullable=False)
cost = Column(Float, nullable=False)
class Bids(Base):
__tablename__ = 'bids'
bid_id = Column(String(64), primary_key=True, autoincrement=False)
project_id = Column(String(64), nullable=False)
quantity = Column(Integer, nullable=False)
start_time = Column(DateTime(timezone=True), nullable=False)
end_time = Column(DateTime(timezone=True), nullable=False)
duration = Column(Integer, nullable=False)
status = Column(String(16), nullable=False, default=statuses.AVAILABLE)
# config_query = Column(String(200))
config_query = Column(sqlalchemy_jsonfield.JSONField(
enforce_string=True, enforce_unicode=False),
nullable=False)
cost = Column(Float, nullable=False)
class HentaiReleases(Base):
__tablename__ = 'hentai_releases'
id = Column(BigInteger, primary_key=True)
state = Column(dlstate_enum, nullable=False, index=True, default='new')
err_str = Column(Text)
source_site = Column(Text, nullable=False,
index=True) # Actual source site
source_id = Column(
Text, nullable=False,
index=True) # ID On source site. Usually (but not always) the item URL
first_seen = Column(DateTime, nullable=False)
posted_at = Column(DateTime, nullable=False, default=datetime.datetime.min)
downloaded_at = Column(DateTime,
nullable=False,
default=datetime.datetime.min)
last_checked = Column(DateTime,
nullable=False,
default=datetime.datetime.min)
deleted = Column(Boolean, default=False, nullable=False)
was_duplicate = Column(Boolean, default=False, nullable=False)
phash_duplicate = Column(Boolean, default=False, nullable=False)
uploaded = Column(Boolean, default=False, nullable=False)
dirstate = Column(dir_type, nullable=False, default="unknown")
origin_name = Column(citext.CIText())
series_name = Column(citext.CIText(), index=True)
additional_metadata = Column(sqlalchemy_jsonfield.JSONField())
fileid = Column(BigInteger, ForeignKey('release_files.id'))
file = relationship('ReleaseFile', backref='hentai_releases')
tags_rel = relationship('HentaiTags',
secondary=hentai_releases_tags_link,
backref=backref("hentai_releases", lazy='dynamic'),
collection_class=set)
tags = association_proxy('tags_rel',
'tag',
creator=HentaiTags.get_or_create)
__table_args__ = (UniqueConstraint('source_site', 'source_id'),
Index('hentai_releases_source_site_id_idx',
'source_site', 'source_id'))
class Bid(Base):
__tablename__ = 'bids'
marketplace_bid_id = orm.Column(
orm.String(64),
primary_key=True,
autoincrement=False,
)
project_id = orm.Column(orm.String(64), nullable=False)
server_quantity = orm.Column(orm.Integer, nullable=False)
start_time = orm.Column(orm.DateTime(timezone=True), nullable=False)
end_time = orm.Column(orm.DateTime(timezone=True), nullable=False)
duration = orm.Column(orm.Integer, nullable=False)
status = orm.Column(orm.String(15), nullable=False, default='available')
server_config_query = orm.Column(sqlalchemy_jsonfield.JSONField(
enforce_string=True, enforce_unicode=False),
nullable=False)
cost = orm.Column(orm.Float, nullable=False)
contracts = orm.relationship('Contract', lazy='dynamic')
class UnitTest(klotio.models.MySQL.Base):
__tablename__ = "unittest"
id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True)
name = sqlalchemy.Column(sqlalchemy.String(64), nullable=False)
data = sqlalchemy.Column(
sqlalchemy.ext.mutable.MutableDict.as_mutable(
sqlalchemy_jsonfield.JSONField(enforce_string=True,enforce_unicode=False)
),
nullable=False,
default=dict
)
sqlalchemy.schema.UniqueConstraint('name', name='label')
def __repr__(self):
return "<UnitTest(name='%s')>" % (self.name)
class Template(Base):
__tablename__ = "template"
template_id = sqlalchemy.Column(sqlalchemy.Integer,
primary_key=True,
autoincrement=True)
name = sqlalchemy.Column(sqlalchemy.String(128), nullable=False)
kind = sqlalchemy.Column(sqlalchemy.Enum("chore", "act"))
data = sqlalchemy.Column(sqlalchemy.ext.mutable.MutableDict.as_mutable(
sqlalchemy_jsonfield.JSONField(enforce_string=True,
enforce_unicode=False)),
nullable=False)
sqlalchemy.schema.UniqueConstraint('name', 'kind', name='label')
def __repr__(self):
return "<Template(name='%s',kind='%s')>" % (self.name, self.kind)
class ScrapeTargets(Base):
__tablename__ = 'scrape_targets'
id = Column(BigInteger, primary_key = True)
site_name = Column(Text, nullable=False)
artist_name = Column(Text, nullable=False, index=True)
uploadeh = Column(Boolean, index = True, default=False, nullable=True)
last_fetched = Column(DateTime, nullable=False, default=datetime.datetime.min)
extra_meta = Column(sqlalchemy_jsonfield.JSONField())
release_cnt = Column(Integer, default='0')
posts = relationship("ArtItem")
in_progress = Column(Boolean, default=False)
__table_args__ = (
UniqueConstraint('site_name', 'artist_name'),
)
class Area(Base):
__tablename__ = "area"
area_id = sqlalchemy.Column(sqlalchemy.Integer,
primary_key=True,
autoincrement=True)
name = sqlalchemy.Column(sqlalchemy.String(64), nullable=False)
status = sqlalchemy.Column(sqlalchemy.String(32), nullable=False)
updated = sqlalchemy.Column(sqlalchemy.Integer)
data = sqlalchemy.Column(sqlalchemy.ext.mutable.MutableDict.as_mutable(
sqlalchemy_jsonfield.JSONField(enforce_string=True,
enforce_unicode=False)),
nullable=False)
sqlalchemy.schema.UniqueConstraint('name', name='label')
def __repr__(self):
return "<Area(name='%s')>" % (self.name)
def _create_dataset_event_table():
op.create_table(
'dataset_event',
sa.Column('id', Integer, primary_key=True, autoincrement=True),
sa.Column('dataset_id', Integer, nullable=False),
sa.Column('extra',
sqlalchemy_jsonfield.JSONField(json=json),
nullable=False,
default={}),
sa.Column('source_task_id', String(250), nullable=True),
sa.Column('source_dag_id', String(250), nullable=True),
sa.Column('source_run_id', String(250), nullable=True),
sa.Column('source_map_index',
sa.Integer(),
nullable=True,
server_default='-1'),
sa.Column('timestamp', TIMESTAMP, nullable=False),
sqlite_autoincrement=True, # ensures PK values not reused
)
op.create_index('idx_dataset_id_timestamp', 'dataset_event',
['dataset_id', 'timestamp'])
class Offer(Base):
__tablename__ = "offers"
marketplace_offer_id = orm.Column(
orm.String(64),
primary_key=True,
autoincrement=False,
)
provider_id = orm.Column(orm.String(64), nullable=False)
creator_id = orm.Column(orm.String(64), nullable=False)
marketplace_date_created = orm.Column(orm.DateTime(timezone=True),
nullable=False)
status = orm.Column(orm.String(15), nullable=False, default="available")
server_id = orm.Column(orm.String(64), nullable=False, unique=True)
start_time = orm.Column(orm.DateTime(timezone=True), nullable=False)
end_time = orm.Column(orm.DateTime(timezone=True), nullable=False)
server_config = orm.Column(
sqlalchemy_jsonfield.JSONField(enforce_string=True,
enforce_unicode=False),
nullable=False,
)
cost = orm.Column(orm.Float, nullable=False)
class BookReleases(Base):
__tablename__ = 'book_releases'
id = Column(BigInteger, primary_key=True)
state = Column(dlstate_enum, nullable=False, index=True, default='new')
err_str = Column(Text)
source_site = Column(Text, nullable=False,
index=True) # Actual source site
source_id = Column(
Text, nullable=False,
index=True) # ID On source site. Usually (but not always) the item URL
first_seen = Column(DateTime, nullable=False)
posted_at = Column(DateTime, nullable=False, default=datetime.datetime.min)
downloaded_at = Column(DateTime,
nullable=False,
default=datetime.datetime.min)
last_checked = Column(DateTime,
nullable=False,
default=datetime.datetime.min)
deleted = Column(Boolean, default=False, nullable=False)
was_duplicate = Column(Boolean, default=False, nullable=False)
phash_duplicate = Column(Boolean, default=False, nullable=False)
uploaded = Column(Boolean, default=False, nullable=False)
dirstate = Column(dir_type, nullable=False, default="unknown")
origin_name = Column(citext.CIText())
series_name = Column(citext.CIText(), index=True)
additional_metadata = Column(sqlalchemy_jsonfield.JSONField())
fileid = Column(BigInteger, ForeignKey('release_files.id'))
file = relationship('ReleaseFile', backref='book_releases')
__table_args__ = (UniqueConstraint('source_site', 'source_id'),
Index('book_releases_source_site_id_idx', 'source_site',
'source_id'))
