def django_db_setup(django_db_setup, django_db_keepdb): database = Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USER, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=CLICKHOUSE_VERIFY, ) if not django_db_keepdb: try: database.drop_database() except: pass if not django_db_keepdb or not database.db_exists: database.create_database() database.migrate("ee.clickhouse.migrations") # Make DELETE / UPDATE synchronous to avoid flaky tests sync_execute("SET mutations_sync = 1") yield if not django_db_keepdb: try: database.drop_database() except: pass
def django_db_setup(django_db_setup, django_db_keepdb): database = Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USER, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=CLICKHOUSE_VERIFY, ) if not django_db_keepdb: try: database.drop_database() except: pass database.create_database() # Create database if it doesn't exist table_count = sync_execute( "SELECT count() FROM system.tables WHERE database = %(database)s", {"database": CLICKHOUSE_DATABASE})[0][0] create_clickhouse_tables(table_count) yield if django_db_keepdb: reset_clickhouse_tables() else: try: database.drop_database() except: pass
def get_database(self) -> Database: return Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USERNAME, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=CLICKHOUSE_VERIFY, )
def handle(self, *args, **options): Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USERNAME, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=False, ).migrate("ee.clickhouse.migrations")
def teardown_databases(self, old_config, **kwargs): Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USERNAME, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=CLICKHOUSE_VERIFY, ).drop_database() super().teardown_databases(old_config, **kwargs)
def setup_databases(self, **kwargs): Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USERNAME, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=CLICKHOUSE_VERIFY, ).migrate("ee.clickhouse.migrations") return super().setup_databases(**kwargs)
def _get_db(db_name): ''' Returns a Database instance using connection information from the command line arguments (optional). ''' db_url = sys.argv[1] if len(sys.argv) > 1 else 'http://localhost:8123/' username = sys.argv[2] if len(sys.argv) > 2 else None password = sys.argv[3] if len(sys.argv) > 3 else None return Database(db_name, db_url, username, password, readonly=True)
def handle(self, *args, **options): try: Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USERNAME, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=False, ).migrate("ee.clickhouse.migrations") print("migration successful") except Exception as e: print(e)
def handle(self, *args, **options): database = Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USER, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=False, ) if options["plan"]: print("List of clickhouse migrations to be applied:") for migration_name in self.get_migrations(database, options["upto"]): print(f"Migration would get applied: {migration_name}") else: print("Clickhouse migrations up to date!") elif options["fake"]: for migration_name in self.get_migrations(database, options["upto"]): print(f"Faked migration: {migration_name}") database.insert([ MigrationHistory( package_name=MIGRATIONS_PACKAGE_NAME, module_name=migration_name, applied=datetime.date.today(), ) ]) print("Migrations done") else: database.migrate(MIGRATIONS_PACKAGE_NAME, options["upto"]) print("Migration successful")
def migrate(self, host, options): database = Database( CLICKHOUSE_DATABASE, db_url=host, username=CLICKHOUSE_USER, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=False, ) if options["plan"]: print("List of clickhouse migrations to be applied:") migrations = list(self.get_migrations(database, options["upto"])) for migration_name, operations in migrations: print(f"Migration would get applied: {migration_name}") for op in operations: sql = getattr(op, "_sql") if options["print_sql"] and sql is not None: print(indent("\n\n".join(sql), " ")) if len(migrations) == 0: print("Clickhouse migrations up to date!") elif options["fake"]: for migration_name, _ in self.get_migrations( database, options["upto"]): print(f"Faked migration: {migration_name}") database.insert([ MigrationHistory( package_name=MIGRATIONS_PACKAGE_NAME, module_name=migration_name, applied=datetime.date.today(), ) ]) print("Migrations done") else: database.migrate(MIGRATIONS_PACKAGE_NAME, options["upto"]) print("Migration successful")
def handle(self, *args, **options): from django.test.runner import DiscoverRunner as TestRunner test_runner = TestRunner(interactive=False) test_runner.setup_databases() test_runner.setup_test_environment() if is_clickhouse_enabled(): from infi.clickhouse_orm import Database # type: ignore from posthog.settings import ( CLICKHOUSE_DATABASE, CLICKHOUSE_HTTP_URL, CLICKHOUSE_PASSWORD, CLICKHOUSE_REPLICATION, CLICKHOUSE_USER, CLICKHOUSE_VERIFY, ) database = Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USER, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=CLICKHOUSE_VERIFY, ) try: database.create_database() except: pass database.migrate("ee.clickhouse.migrations", replicated=CLICKHOUSE_REPLICATION)
def handle(self, *args, **options): if not TEST: raise ValueError( "TEST environment variable needs to be set for this command to function" ) from django.test.runner import DiscoverRunner as TestRunner test_runner = TestRunner(interactive=False) test_runner.setup_databases() test_runner.setup_test_environment() from infi.clickhouse_orm import Database from posthog.settings import ( CLICKHOUSE_CLUSTER, CLICKHOUSE_DATABASE, CLICKHOUSE_HTTP_URL, CLICKHOUSE_PASSWORD, CLICKHOUSE_REPLICATION, CLICKHOUSE_USER, CLICKHOUSE_VERIFY, ) database = Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USER, password=CLICKHOUSE_PASSWORD, cluster=CLICKHOUSE_CLUSTER, verify_ssl_cert=CLICKHOUSE_VERIFY, ) try: database.create_database() except: pass database.migrate("ee.clickhouse.migrations", replicated=CLICKHOUSE_REPLICATION)
def get_fragments(filename): ''' Converts a text file at the given path to a generator of Fragment instances. ''' from os import path document = path.splitext(path.basename(filename))[0] idx = 0 for word, stem in parse_file(filename): idx += 1 yield Fragment(document=document, idx=idx, word=word, stem=stem) print('{} - {} words'.format(filename, idx)) if __name__ == '__main__': # Load NLTK data if necessary nltk.download('punkt') nltk.download('wordnet') # Initialize database db = Database('default') db.create_table(Fragment) # Load files from the command line or everything under ebooks/ filenames = sys.argv[1:] or glob('ebooks/*.txt') for filename in filenames: db.insert(get_fragments(filename), batch_size=100000)
from infi.clickhouse_orm import Database Database('default').migrate('clicks.migrations')
import psutil, time, datetime from infi.clickhouse_orm import Database from models import CPUStats db = Database('demo') db.create_table(CPUStats) psutil.cpu_percent(percpu=True) # first sample should be discarded while True: time.sleep(1) stats = psutil.cpu_percent(percpu=True) timestamp = datetime.datetime.now() print(timestamp) db.insert([ CPUStats(timestamp=timestamp, cpu_id=cpu_id, cpu_percent=cpu_percent) for cpu_id, cpu_percent in enumerate(stats) ])
from infi.clickhouse_orm import Database, F from models import CPUStats db = Database('demo') queryset = CPUStats.objects_in(db) total = queryset.filter(CPUStats.cpu_id == 1).count() busy = queryset.filter(CPUStats.cpu_id == 1, CPUStats.cpu_percent > 95).count() print('CPU 1 was busy {:.2f}% of the time'.format(busy * 100.0 / total)) # Calculate the average usage per CPU for row in queryset.aggregate(CPUStats.cpu_id, average=F.avg(CPUStats.cpu_percent)): print('CPU {row.cpu_id}: {row.average:.2f}%'.format(row=row))
word = word + Style.RESET_ALL text.append(word) return ' '.join(text) def find(db, text): ''' Performs the search for the given text, and prints out the matches. ''' stems = prepare_search_terms(text) query = build_query(db, stems) print('\n' + Fore.MAGENTA + str(query) + Style.RESET_ALL + '\n') for match in query: text = get_matching_text(db, match.document, match.idx, match.idx + len(stems) - 1) print(Fore.CYAN + match.document + ':' + Style.RESET_ALL, text) if __name__ == '__main__': # Initialize colored output init() # Initialize database db = Database('default') # Search text = ' '.join(sys.argv[1:]) if text: find(db, text)