log_level = logging.INFO
if verbose:
log_level = logging.DEBUG
logging.basicConfig(level=log_level,
format='%(asctime)s %(levelname)-6s %(message)s',
datefmt='%Y-%m-%dT%H:%M:%S')
if tables:
tables = tables.split(',')
else:
properties = load_properties('/u/apps/camus/shared/camus.properties')
tables = fetch_kafka_trekkie_topics(properties['kafka.brokers'])
hive = HiveUtils(database, hive_options)
for table in tables:
hdfs_location = table
table = table.replace('.', '_').replace('-', '_')
if not hive.table_exists(table):
if dry_run:
logging.info(str(hive_trekkie_create_table_stmt(table, hdfs_location)))
else:
hive.table_create(table, hdfs_location)
if dry_run:
logging.info(str(hive.get_missing_partitions_ddl(table)))
else:
hive.add_missing_partitions(table)
