class DISKStats(Model):
event_date = DateField()
timestamp = DateTimeField()
total = Float32Field()
used = Float32Field()
engine = MergeTree('event_date', ('total', 'used', 'timestamp'))
class CPUStats(Model):
event_date = DateField()
timestamp = DateTimeField()
cpu_id = UInt16Field()
cpu_percent = Float32Field()
engine = MergeTree('event_date', ('cpu_id', 'cpu_percent', 'timestamp'))
class Events(Model):
ts = DateTimeField()
userId = UInt32Field()
sessionId = UInt32Field()
page = FixedStringField(50)
auth = FixedStringField(10)
method = FixedStringField(7)
status = UInt16Field()
level = Enum8Field(Level)
itemInSession = UInt16Field()
location = NullableField(StringField())
userAgent = NullableField(StringField())
lastName = NullableField(FixedStringField(50))
firstName = NullableField(FixedStringField(50))
registration = DateTimeField()
gender = Enum8Field(Gender)
artist = NullableField(StringField())
song = NullableField(StringField())
length = Float32Field()
engine = MergeTree(
partition_key=('auth', ),
order_by=(
'userId',
'ts',
'page',
'gender',
),
)
class LoggingMsg(Model):
timestamp = DateTime64Field(codec="Delta,ZSTD")
msg = StringField()
level = UInt8Field(codec="Delta, LZ4")
payload = StringField(default="")
engine = MergeTree("timestamp", order_by=("timestamp", ))
class CONNStats(Model):
event_date = DateField()
timestamp = DateTimeField()
protocol = StringField()
src_addr = StringField()
src_port = Float32Field()
dst_addr = StringField()
dst_port = Float32Field()
qry_name = StringField()
engine = MergeTree('event_date',
('timestamp', 'protocol', 'src_addr', 'src_port',
'dst_addr', 'dst_port', 'qry_name'))
class Rates(Model):
event_date = DateField()
event_time = DateTimeField()
code = StringField()
time = Int32Field()
close = Float32Field()
high = Float32Field()
low = Float32Field()
open = Float32Field()
volumefrom = Float32Field()
volumeto = Float32Field()
engine = MergeTree('event_date', ['event_date', 'time', 'code'])
class ClickhouseAllFields(chm.Model):
id = chf.Int64Field()
timestamp = chf.DateTimeField()
timestamp_date = chf.DateField()
string_field = chf.StringField()
intfield = chf.Int32Field()
floatfield = chf.Float32Field()
null_field = chf.NullableField(chf.StringField())
enum_field = chf.Enum16Field(enum_)
array_field = chf.ArrayField(chf.Int16Field())
engine = MergeTree('timestamp_date', ['id'])
class DepthSnapshot(Model):
timestamp = DateTime64Field(codec="Delta,ZSTD")
last_update_id = UInt64Field()
bids_quantity = ArrayField(Float64Field())
bids_price = ArrayField(Float64Field())
asks_quantity = ArrayField(Float64Field())
asks_price = ArrayField(Float64Field())
symbol = LowCardinalityField(StringField())
engine = MergeTree(
partition_key=("symbol", ),
order_by=("timestamp", "last_update_id"),
)
class IOCStats(Model):
event_date = DateField()
timestamp = DateTimeField()
protocol = StringField()
src_addr = StringField()
src_port = Float32Field()
dst_addr = StringField()
dst_port = Float32Field()
qry_name = StringField()
indicator = StringField()
name = StringField()
references = StringField()
engine = MergeTree(
'event_date',
('timestamp', 'protocol', 'src_addr', 'src_port', 'dst_addr',
'dst_port', 'qry_name', 'indicator', 'name', 'references'))
class OPEN_PORTS(Model):
event_date = DateField()
timestamp = DateTimeField()
os = StringField()
os_v = StringField()
srv = StringField()
addr = StringField()
port = StringField()
product = StringField()
version = StringField()
descr = StringField()
vdesc = StringField()
title = StringField()
cvelist = StringField()
score = Float32Field()
engine = MergeTree('event_date',
('os', 'os_v', 'srv', 'addr', 'port', 'product',
'version', 'descr', 'vdesc', 'score'))
class Events(Model):
ts = DateTimeField()
userId = StringField()
sessionId = UInt16Field()
page = FixedStringField(30)
auth = FixedStringField(20)
method = FixedStringField(10)
status = UInt16Field()
level = FixedStringField(10)
itemSession = UInt16Field()
location = NullableField(StringField())
userAgent = NullableField(StringField())
lastname = NullableField(FixedStringField(300))
firstname = NullableField(FixedStringField(300))
registration = UInt64Field()
gender = NullableField(FixedStringField(2))
artist = NullableField(FixedStringField(500))
song = NullableField(StringField())
length = Float32Field()
engine = MergeTree(partition_key=('level', ), order_by=(
'ts',
'userId',
))
class Length(Model):
query_date = DateField()
timestamp = DateTimeField()
length = UInt32Field()
engine = MergeTree('query_date', ('timestamp', ))
class Mask(Model):
query_date = DateField()
timestamp = DateTimeField()
mask = StringField()
engine = MergeTree('query_date', ('timestamp', 'mask'))
class Domains(Model):
query_date = DateField()
timestamp = DateTimeField()
domain_name = StringField()
engine = MergeTree('query_date', ('timestamp', 'domain_name'))
class MigrationHistory(ClickHouseModel):
"""
A model for storing which migrations were already applied to database.
"""
db_alias = StringField()
package_name = StringField()
module_name = StringField()
applied = DateField()
engine = MergeTree('applied', ('db_alias', 'package_name', 'module_name'))
@classmethod
def set_migration_applied(cls, db_alias: str, migrations_package: str,
name: str) -> None:
"""
Sets migration apply status
:param db_alias: Database alias migration is applied to
:param migrations_package: Package migration is stored in
:param name: Migration name
:return: None
"""
# Ensure that table for migration storing is created
for db_name in cls.migrate_non_replicated_db_aliases:
connections[db_name].create_table(cls)
cls.objects.create(db_alias=db_alias,
package_name=migrations_package,
module_name=name,
applied=datetime.date.today())
@classmethod
def get_applied_migrations(cls, db_alias: str,
migrations_package: str) -> Set[str]:
"""
Returns applied migrations names
:param db_alias: Database alias, to check
:param migrations_package: Package name to check
:return: Set of migration names
"""
qs = cls.objects.filter(package_name=migrations_package,
db_alias=db_alias).only('module_name')
try:
return set(obj.module_name for obj in qs)
except ServerError as ex:
# Database doesn't exist or table doesn't exist
# I added string check, when code parsing broke in infi.clickouse_orm
# See https://github.com/Infinidat/infi.clickhouse_orm/issues/108
if ex.code in {
81, 60
} or 'Code: 60' in ex.message or 'Code: 81,' in ex.message:
return set()
raise ex
except DatabaseException as ex:
# If database doesn't exist no migrations are applied
# This prevents readonly=True + db_exists=False infi exception
if str(
ex
) == 'Database does not exist, and cannot be created under readonly connection':
return set()
raise ex
@classmethod
def table_name(cls):
return 'django_clickhouse_migrations'
{% if cookiecutter.db == 'django' %}
from django.db import models
class {{cookiecutter.model_name}}(models.Model):
pass
{% elif cookiecutter.db == 'elastic' %}
class {{cookiecutter.model_name}}(Document):
class Index:
name = '{{cookiecutter.model_name.lower}}'
{% elif cookiecutter.db == 'clickhouse' %}
from infi.clickhouse_orm.database import Database
from infi.clickhouse_orm.models import Model
from infi.clickhouse_orm.fields import *
from infi.clickhouse_orm.engines import MergeTree
class {{cookiecutter.model_name}}(Model):
engine = MergeTree()
{% endif %}
