def migrate(self, host, options): database = Database( CLICKHOUSE_DATABASE, db_url=host, username=CLICKHOUSE_USER, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=False, ) if options["plan"]: print("List of clickhouse migrations to be applied:") migrations = list(self.get_migrations(database, options["upto"])) for migration_name, operations in migrations: print(f"Migration would get applied: {migration_name}") for op in operations: sql = getattr(op, "_sql") if options["print_sql"] and sql is not None: print(indent("\n\n".join(sql), " ")) if len(migrations) == 0: print("Clickhouse migrations up to date!") elif options["fake"]: for migration_name, _ in self.get_migrations( database, options["upto"]): print(f"Faked migration: {migration_name}") database.insert([ MigrationHistory( package_name=MIGRATIONS_PACKAGE_NAME, module_name=migration_name, applied=datetime.date.today(), ) ]) print("Migrations done") else: database.migrate(MIGRATIONS_PACKAGE_NAME, options["upto"]) print("Migration successful")
def handle(self, *args, **options): database = Database( CLICKHOUSE_DATABASE, db_url=CLICKHOUSE_HTTP_URL, username=CLICKHOUSE_USER, password=CLICKHOUSE_PASSWORD, verify_ssl_cert=False, ) if options["plan"]: print("List of clickhouse migrations to be applied:") for migration_name in self.get_migrations(database, options["upto"]): print(f"Migration would get applied: {migration_name}") else: print("Clickhouse migrations up to date!") elif options["fake"]: for migration_name in self.get_migrations(database, options["upto"]): print(f"Faked migration: {migration_name}") database.insert([ MigrationHistory( package_name=MIGRATIONS_PACKAGE_NAME, module_name=migration_name, applied=datetime.date.today(), ) ]) print("Migrations done") else: database.migrate(MIGRATIONS_PACKAGE_NAME, options["upto"]) print("Migration successful")
import psutil, time, datetime from infi.clickhouse_orm import Database from models import CPUStats db = Database('demo') db.create_table(CPUStats) psutil.cpu_percent(percpu=True) # first sample should be discarded while True: time.sleep(1) stats = psutil.cpu_percent(percpu=True) timestamp = datetime.datetime.now() print(timestamp) db.insert([ CPUStats(timestamp=timestamp, cpu_id=cpu_id, cpu_percent=cpu_percent) for cpu_id, cpu_percent in enumerate(stats) ])
def get_fragments(filename): ''' Converts a text file at the given path to a generator of Fragment instances. ''' from os import path document = path.splitext(path.basename(filename))[0] idx = 0 for word, stem in parse_file(filename): idx += 1 yield Fragment(document=document, idx=idx, word=word, stem=stem) print('{} - {} words'.format(filename, idx)) if __name__ == '__main__': # Load NLTK data if necessary nltk.download('punkt') nltk.download('wordnet') # Initialize database db = Database('default') db.create_table(Fragment) # Load files from the command line or everything under ebooks/ filenames = sys.argv[1:] or glob('ebooks/*.txt') for filename in filenames: db.insert(get_fragments(filename), batch_size=100000)