def migrate(self, host, options):
database = Database(
CLICKHOUSE_DATABASE,
db_url=host,
username=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=False,
)
if options["plan"]:
print("List of clickhouse migrations to be applied:")
migrations = list(self.get_migrations(database, options["upto"]))
for migration_name, operations in migrations:
print(f"Migration would get applied: {migration_name}")
for op in operations:
sql = getattr(op, "_sql")
if options["print_sql"] and sql is not None:
print(indent("\n\n".join(sql), " "))
if len(migrations) == 0:
print("Clickhouse migrations up to date!")
elif options["fake"]:
for migration_name, _ in self.get_migrations(
database, options["upto"]):
print(f"Faked migration: {migration_name}")
database.insert([
MigrationHistory(
package_name=MIGRATIONS_PACKAGE_NAME,
module_name=migration_name,
applied=datetime.date.today(),
)
])
print("Migrations done")
else:
database.migrate(MIGRATIONS_PACKAGE_NAME, options["upto"])
print("Migration successful")
def handle(self, *args, **options):
database = Database(
CLICKHOUSE_DATABASE,
db_url=CLICKHOUSE_HTTP_URL,
username=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
verify_ssl_cert=False,
)
if options["plan"]:
print("List of clickhouse migrations to be applied:")
for migration_name in self.get_migrations(database,
options["upto"]):
print(f"Migration would get applied: {migration_name}")
else:
print("Clickhouse migrations up to date!")
elif options["fake"]:
for migration_name in self.get_migrations(database,
options["upto"]):
print(f"Faked migration: {migration_name}")
database.insert([
MigrationHistory(
package_name=MIGRATIONS_PACKAGE_NAME,
module_name=migration_name,
applied=datetime.date.today(),
)
])
print("Migrations done")
else:
database.migrate(MIGRATIONS_PACKAGE_NAME, options["upto"])
print("Migration successful")
import psutil, time, datetime
from infi.clickhouse_orm import Database
from models import CPUStats
db = Database('demo')
db.create_table(CPUStats)
psutil.cpu_percent(percpu=True) # first sample should be discarded
while True:
time.sleep(1)
stats = psutil.cpu_percent(percpu=True)
timestamp = datetime.datetime.now()
print(timestamp)
db.insert([
CPUStats(timestamp=timestamp, cpu_id=cpu_id, cpu_percent=cpu_percent)
for cpu_id, cpu_percent in enumerate(stats)
])
def get_fragments(filename):
'''
Converts a text file at the given path to a generator
of Fragment instances.
'''
from os import path
document = path.splitext(path.basename(filename))[0]
idx = 0
for word, stem in parse_file(filename):
idx += 1
yield Fragment(document=document, idx=idx, word=word, stem=stem)
print('{} - {} words'.format(filename, idx))
if __name__ == '__main__':
# Load NLTK data if necessary
nltk.download('punkt')
nltk.download('wordnet')
# Initialize database
db = Database('default')
db.create_table(Fragment)
# Load files from the command line or everything under ebooks/
filenames = sys.argv[1:] or glob('ebooks/*.txt')
for filename in filenames:
db.insert(get_fragments(filename), batch_size=100000)
