def django_db_setup(django_db_setup, django_db_keepdb):
database = Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=CLICKHOUSE_VERIFY,
)
if not django_db_keepdb:
try:
database.drop_database()
except:
pass
if not django_db_keepdb or not database.db_exists:
database.create_database()
database.migrate("ee.clickhouse.migrations")
# Make DELETE / UPDATE synchronous to avoid flaky tests
sync_execute("SET mutations_sync = 1")
yield
if not django_db_keepdb:
try:
database.drop_database()
except:
pass
def django_db_setup(django_db_setup, django_db_keepdb):
database = Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=CLICKHOUSE_VERIFY,
)
if not django_db_keepdb:
try:
database.drop_database()
except:
pass
database.create_database() # Create database if it doesn't exist
table_count = sync_execute(
"SELECT count() FROM system.tables WHERE database = %(database)s",
{"database": CLICKHOUSE_DATABASE})[0][0]
create_clickhouse_tables(table_count)
yield
if django_db_keepdb:
reset_clickhouse_tables()
else:
try:
database.drop_database()
except:
pass
def get_database(self) -> Database:
return Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USERNAME,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=CLICKHOUSE_VERIFY,
)
def handle(self, *args, **options):
Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USERNAME,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=False,
).migrate("ee.clickhouse.migrations")
def teardown_databases(self, old_config, **kwargs):
Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USERNAME,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=CLICKHOUSE_VERIFY,
).drop_database()
super().teardown_databases(old_config, **kwargs)
def setup_databases(self, **kwargs):
Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USERNAME,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=CLICKHOUSE_VERIFY,
).migrate("ee.clickhouse.migrations")
return super().setup_databases(**kwargs)
def _get_db(db_name):
'''
Returns a Database instance using connection information
from the command line arguments (optional).
'''
db_url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8123/'
username = sys.argv[2] if len(sys.argv) > 2 else None
password = sys.argv[3] if len(sys.argv) > 3 else None
return Database(db_name, db_url, username, password, readonly=True)
def handle(self, *args, **options):
try:
Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USERNAME,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=False,
).migrate("ee.clickhouse.migrations")
print("migration successful")
except Exception as e:
print(e)
def handle(self, *args, **options):
database = Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=False,
)
if options["plan"]:
print("List of clickhouse migrations to be applied:")
for migration_name in self.get_migrations(database,
options["upto"]):
print(f"Migration would get applied: {migration_name}")
else:
print("Clickhouse migrations up to date!")
elif options["fake"]:
for migration_name in self.get_migrations(database,
options["upto"]):
print(f"Faked migration: {migration_name}")
database.insert([
MigrationHistory(
package_name=MIGRATIONS_PACKAGE_NAME,
module_name=migration_name,
applied=datetime.date.today(),
)
])
print("Migrations done")
else:
database.migrate(MIGRATIONS_PACKAGE_NAME, options["upto"])
print("Migration successful")
def migrate(self, host, options):
database = Database(
CLICKHOUSE_DATABASE,
db_url=host,
username=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=False,
)
if options["plan"]:
print("List of clickhouse migrations to be applied:")
migrations = list(self.get_migrations(database, options["upto"]))
for migration_name, operations in migrations:
print(f"Migration would get applied: {migration_name}")
for op in operations:
sql = getattr(op, "_sql")
if options["print_sql"] and sql is not None:
print(indent("\n\n".join(sql), " "))
if len(migrations) == 0:
print("Clickhouse migrations up to date!")
elif options["fake"]:
for migration_name, _ in self.get_migrations(
database, options["upto"]):
print(f"Faked migration: {migration_name}")
database.insert([
MigrationHistory(
package_name=MIGRATIONS_PACKAGE_NAME,
module_name=migration_name,
applied=datetime.date.today(),
)
])
print("Migrations done")
else:
database.migrate(MIGRATIONS_PACKAGE_NAME, options["upto"])
print("Migration successful")
def handle(self, *args, **options):
from django.test.runner import DiscoverRunner as TestRunner
test_runner = TestRunner(interactive=False)
test_runner.setup_databases()
test_runner.setup_test_environment()
if is_clickhouse_enabled():
from infi.clickhouse_orm import Database # type: ignore
from posthog.settings import (
CLICKHOUSE_DATABASE,
CLICKHOUSE_HTTP_URL,
CLICKHOUSE_PASSWORD,
CLICKHOUSE_REPLICATION,
CLICKHOUSE_USER,
CLICKHOUSE_VERIFY,
)
database = Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=CLICKHOUSE_VERIFY,
)
try:
database.create_database()
except:
pass
database.migrate("ee.clickhouse.migrations",
replicated=CLICKHOUSE_REPLICATION)
def handle(self, *args, **options):
if not TEST:
raise ValueError(
"TEST environment variable needs to be set for this command to function"
)
from django.test.runner import DiscoverRunner as TestRunner
test_runner = TestRunner(interactive=False)
test_runner.setup_databases()
test_runner.setup_test_environment()
from infi.clickhouse_orm import Database
from posthog.settings import (
CLICKHOUSE_CLUSTER,
CLICKHOUSE_DATABASE,
CLICKHOUSE_HTTP_URL,
CLICKHOUSE_PASSWORD,
CLICKHOUSE_REPLICATION,
CLICKHOUSE_USER,
CLICKHOUSE_VERIFY,
)
database = Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
cluster=CLICKHOUSE_CLUSTER,
verify_ssl_cert=CLICKHOUSE_VERIFY,
)
try:
database.create_database()
except:
pass
database.migrate("ee.clickhouse.migrations",
replicated=CLICKHOUSE_REPLICATION)
def get_fragments(filename):
'''
Converts a text file at the given path to a generator
of Fragment instances.
'''
from os import path
document = path.splitext(path.basename(filename))[0]
idx = 0
for word, stem in parse_file(filename):
idx += 1
yield Fragment(document=document, idx=idx, word=word, stem=stem)
print('{} - {} words'.format(filename, idx))
if __name__ == '__main__':
# Load NLTK data if necessary
nltk.download('punkt')
nltk.download('wordnet')
# Initialize database
db = Database('default')
db.create_table(Fragment)
# Load files from the command line or everything under ebooks/
filenames = sys.argv[1:] or glob('ebooks/*.txt')
for filename in filenames:
db.insert(get_fragments(filename), batch_size=100000)
from infi.clickhouse_orm import Database
Database('default').migrate('clicks.migrations')
import psutil, time, datetime
from infi.clickhouse_orm import Database
from models import CPUStats
db = Database('demo')
db.create_table(CPUStats)
psutil.cpu_percent(percpu=True) # first sample should be discarded
while True:
time.sleep(1)
stats = psutil.cpu_percent(percpu=True)
timestamp = datetime.datetime.now()
print(timestamp)
db.insert([
CPUStats(timestamp=timestamp, cpu_id=cpu_id, cpu_percent=cpu_percent)
for cpu_id, cpu_percent in enumerate(stats)
])
from infi.clickhouse_orm import Database, F
from models import CPUStats
db = Database('demo')
queryset = CPUStats.objects_in(db)
total = queryset.filter(CPUStats.cpu_id == 1).count()
busy = queryset.filter(CPUStats.cpu_id == 1, CPUStats.cpu_percent > 95).count()
print('CPU 1 was busy {:.2f}% of the time'.format(busy * 100.0 / total))
# Calculate the average usage per CPU
for row in queryset.aggregate(CPUStats.cpu_id,
average=F.avg(CPUStats.cpu_percent)):
print('CPU {row.cpu_id}: {row.average:.2f}%'.format(row=row))
word = word + Style.RESET_ALL
text.append(word)
return ' '.join(text)
def find(db, text):
'''
Performs the search for the given text, and prints out the matches.
'''
stems = prepare_search_terms(text)
query = build_query(db, stems)
print('\n' + Fore.MAGENTA + str(query) + Style.RESET_ALL + '\n')
for match in query:
text = get_matching_text(db, match.document, match.idx,
match.idx + len(stems) - 1)
print(Fore.CYAN + match.document + ':' + Style.RESET_ALL, text)
if __name__ == '__main__':
# Initialize colored output
init()
# Initialize database
db = Database('default')
# Search
text = ' '.join(sys.argv[1:])
if text:
find(db, text)
