def __init__(self, data_dict, dialect, db_name='default', interpreter=None, request=None):
self.name = data_dict['table_name']
if 'partition_files' in data_dict:
self.partition_files = data_dict['partition_files']
else:
self.partition_files = None
self.filename = data_dict['data_file']
self.create_sql = data_dict['create_sql'].strip()
self.insert_sql = data_dict.get('insert_sql')
self.dialect = dialect
self.db_name = db_name
self.interpreter = interpreter
self.request = request
self.columns = data_dict.get('columns')
self.is_transactional = data_dict.get('transactional')
# Sanity check
self._data_dir = LOCAL_EXAMPLES_DATA_DIR.get()
if self.partition_files:
for partition_spec, filename in list(self.partition_files.items()):
filepath = os.path.join(self._data_dir, filename)
self.partition_files[partition_spec] = filepath
self._check_file_contents(filepath)
else:
self._contents_file = os.path.join(self._data_dir, self.filename)
self._check_file_contents(self._contents_file)
def install_queries(self, django_user, dialect, interpreter=None):
design_file = open(os.path.join(LOCAL_EXAMPLES_DATA_DIR.get(), 'queries.json'))
design_list = json.load(design_file)
design_file.close()
# Filter query list to HiveServer2 vs other interfaces
app_type = HQL if dialect == 'hive' else IMPALA if dialect == 'impala' else RDBMS
design_list = [d for d in design_list if int(d['type']) == app_type]
if app_type == RDBMS:
design_list = [d for d in design_list if dialect in d['dialects']]
if self.queries is None: # Manual install
design_list = [d for d in design_list if not d.get('auto_load_only')]
if self.queries: # Automated install
design_list = [d for d in design_list if d['name'] in self.queries]
if not design_list:
raise InstallException(_('No %s queries are available as samples') % dialect)
for design_dict in design_list:
design = SampleQuery(design_dict)
try:
design.install(django_user, interpreter=interpreter)
self.successes.append(_('Query %s %s installed.') % (design_dict['name'], dialect))
except Exception as ex:
msg = str(ex)
LOG.error(msg)
self.errors.append(_('Could not install %s query: %s') % (dialect, msg))
def _install_tables(self, django_user, app_name, db_name, tables, interpreter=None):
data_dir = LOCAL_EXAMPLES_DATA_DIR.get()
table_file = open(os.path.join(data_dir, tables))
table_list = json.load(table_file)
table_file.close()
for table_dict in table_list:
table = SampleTable(table_dict, app_name, db_name, interpreter=interpreter)
try:
table.install(django_user)
except Exception as ex:
raise InstallException(_('Could not install table: %s') % ex)
def _install_queries(self, django_user, app_name, interpreter=None):
design_file = open(os.path.join(LOCAL_EXAMPLES_DATA_DIR.get(), 'designs.json'))
design_list = json.load(design_file)
design_file.close()
# Filter design list to app-specific designs
app_type = HQL if app_name == 'beeswax' else IMPALA
design_list = [d for d in design_list if int(d['type']) == app_type]
for design_dict in design_list:
design = SampleQuery(design_dict)
try:
design.install(django_user, interpreter=interpreter)
except Exception as ex:
raise InstallException(_('Could not install %s query: %s') % (app_name, ex))
def install_tables(self, django_user, dialect, db_name, tables, interpreter=None, request=None):
data_dir = LOCAL_EXAMPLES_DATA_DIR.get()
table_file = open(os.path.join(data_dir, tables))
table_list = json.load(table_file)
table_file.close()
table_list = [table_dict for table_dict in table_list if dialect in table_dict.get('dialects', [dialect])]
if not table_list:
raise InstallException(_('No %s tables are available as samples') % dialect)
for table_dict in table_list:
if self.tables is None or table_dict['table_name'] in self.tables:
full_name = '%s.%s' % (db_name, table_dict['table_name'])
try:
table = SampleTable(table_dict, dialect, db_name, interpreter=interpreter, request=request)
if table.install(django_user):
self.successes.append(_('Table %s installed.') % full_name)
except Exception as ex:
msg = str(ex)
LOG.error(msg)
self.errors.append(_('Could not install table %s: %s') % (full_name, msg))
def __init__(self, data_dict, app_name, db_name='default', interpreter=None):
self.name = data_dict['table_name']
if 'partition_files' in data_dict:
self.partition_files = data_dict['partition_files']
else:
self.partition_files = None
self.filename = data_dict['data_file']
self.hql = data_dict['create_hql']
self.query_server = get_query_server_config(app_name, connector=interpreter)
self.app_name = app_name
self.db_name = db_name
self.columns = data_dict.get('columns')
self.is_transactional = data_dict.get('transactional')
# Sanity check
self._data_dir = LOCAL_EXAMPLES_DATA_DIR.get()
if self.partition_files:
for partition_spec, filename in list(self.partition_files.items()):
filepath = os.path.join(self._data_dir, filename)
self.partition_files[partition_spec] = filepath
self._check_file_contents(filepath)
else:
self._contents_file = os.path.join(self._data_dir, self.filename)
self._check_file_contents(self._contents_file)