def load_partition(self, django_user, partition_spec, filepath, columns):
if has_concurrency_support() and self.is_transactional:
with open(filepath) as f:
hql = \
"""
INSERT INTO TABLE %(tablename)s
PARTITION (%(partition_spec)s)
VALUES %(values)s
""" % {
'tablename': self.name,
'partition_spec': partition_spec,
'values': self._get_sql_insert_values(f, columns)
}
else:
# Upload data found at filepath to HDFS home of user, the load intto a specific partition
LOAD_PARTITION_HQL = \
"""
ALTER TABLE %(tablename)s ADD PARTITION(%(partition_spec)s) LOCATION '%(filepath)s'
"""
partition_dir = self._get_partition_dir(partition_spec)
hdfs_root_destination = self._get_hdfs_root_destination(django_user, subdir=partition_dir)
filename = filepath.split('/')[-1]
hdfs_file_destination = self._upload_to_hdfs(django_user, filepath, hdfs_root_destination, filename)
hql = LOAD_PARTITION_HQL % {'tablename': self.name, 'partition_spec': partition_spec, 'filepath': hdfs_root_destination}
self._load_data_to_table(django_user, hql)
def load(self, django_user):
if has_concurrency_support() and self.is_transactional:
with open(self._contents_file) as f:
hql = \
"""
INSERT INTO TABLE %(tablename)s
VALUES %(values)s
""" % {
'tablename': self.name,
'values': self._get_sql_insert_values(f)
}
else:
# Upload data to HDFS home of user then load (aka move) it into the Hive table (in the Hive metastore in HDFS).
hdfs_root_destination = self._get_hdfs_root_destination(django_user)
hdfs_file_destination = self._upload_to_hdfs(django_user, self._contents_file, hdfs_root_destination)
hql = \
"""
LOAD DATA INPATH
'%(filename)s' OVERWRITE INTO TABLE %(tablename)s
""" % {
'tablename': self.name,
'filename': hdfs_file_destination
}
self._load_data_to_table(django_user, hql)
def install(self, django_user):
if has_concurrency_support() and not self.is_transactional:
LOG.info('Skipping table %s as non transactional' % self.name)
return
if not (has_concurrency_support() and self.is_transactional) and not cluster.get_hdfs():
raise PopupException('Requiring a File System to load its data')
self.create(django_user)
if self.partition_files:
for partition_spec, filepath in list(self.partition_files.items()):
self.load_partition(django_user, partition_spec, filepath, columns=self.columns)
else:
self.load(django_user)
return True
def load(self, django_user):
inserts = []
if (self.dialect not in ('hive', 'impala') or has_concurrency_support()) and self.is_transactional:
with open(self._contents_file) as f:
if self.insert_sql:
sql_insert = self.insert_sql
else:
sql_insert = """
INSERT INTO TABLE %(tablename)s
VALUES %(values)s
"""
values = self._get_sql_insert_values(f)
for value in values:
inserts.append(
sql_insert % {
'tablename': self.name,
'values': value
}
)
else:
# Upload data to HDFS home of user then load (aka move) it into the Hive table (in the Hive metastore in HDFS).
hdfs_root_destination = self._get_hdfs_root_destination(django_user)
hdfs_file_destination = self._upload_to_hdfs(django_user, self._contents_file, hdfs_root_destination)
hql = """
LOAD DATA INPATH
'%(filename)s' OVERWRITE INTO TABLE %(tablename)s
""" % {
'tablename': self.name,
'filename': hdfs_file_destination
}
inserts.append(hql)
for insert in inserts:
self._load_data_to_table(django_user, insert)
def handle(self, *args, **options):
if args:
dialect = args[0]
db_name = args[1] if len(args) > 1 else 'default'
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
request = None
else:
dialect = options['dialect']
db_name = options.get('db_name', 'default')
interpreter = options.get(
'interpreter'
) # Only when connectors are enabled. Later will deprecate `dialect`.
user = options['user']
request = options['request']
tables = 'tables_standard.json' if dialect not in (
'hive',
'impala') else ('tables_transactional.json'
if has_concurrency_support() else 'tables.json')
exception = None
self.successes = []
self.errors = []
try:
sample_user = install_sample_user(
user
) # Documents will belong to the sample user but we run the SQL as the current user
self.install_queries(sample_user, dialect, interpreter=interpreter)
self.install_tables(user,
dialect,
db_name,
tables,
interpreter=interpreter,
request=request)
except Exception as ex:
exception = ex
if exception is not None:
pretty_msg = None
if "Permission denied" in str(exception):
pretty_msg = _(
"Permission denied. Please check with your system administrator."
)
if pretty_msg is not None:
raise PopupException(pretty_msg)
else:
raise exception
return self.successes, self.errors
def _install_queries(self, django_user, app_name):
design_file = file(
os.path.join(beeswax.conf.LOCAL_EXAMPLES_DATA_DIR.get(),
'designs.json'))
design_list = json.load(design_file)
design_file.close()
# Filter design list to app-specific designs
app_type = HQL if app_name == 'beeswax' else IMPALA
design_list = [d for d in design_list if int(d['type']) == app_type]
for design_dict in design_list:
if not has_concurrency_support(
) or design_dict['name'] != 'Sample: Customers':
design = SampleQuery(design_dict)
try:
design.install(django_user)
except Exception as ex:
raise InstallException(_('Could not install query: %s') % ex)
def handle(self, *args, **options):
if args:
app_name = args[0]
db_name = args[1] if len(args) > 1 else 'default'
user = User.objects.get(username=pwd.getpwuid(os.getuid()).pw_name)
else:
app_name = options['app_name']
db_name = options.get('db_name', 'default')
user = options['user']
tables = options['tables'] if 'tables' in options else (
'tables_transactional.json'
if has_concurrency_support() else 'tables.json')
exception = None
# Documents will belong to this user but we run the install as the current user
try:
sample_user = install_sample_user()
self._install_queries(sample_user, app_name)
self._install_tables(user, app_name, db_name, tables)
except Exception as ex:
exception = ex
Document.objects.sync()
if exception is not None:
pretty_msg = None
if "AlreadyExistsException" in exception.message:
pretty_msg = _("SQL table examples already installed.")
if "Permission denied" in exception.message:
pretty_msg = _(
"Permission denied. Please check with your system administrator."
)
if pretty_msg is not None:
raise PopupException(pretty_msg)
else:
raise exception
