class ColumnTypeForm(DependencyAwareForm):
"""
Form used to specify a column during table creation
"""
dependencies = [
("column_type", "array", "array_type"),
("column_type", "map", "map_key_type"),
("column_type", "map", "map_value_type"),
]
column_name = common.HiveIdentifierField(label=_t('Column Name'),
required=True)
column_type = forms.ChoiceField(label=_t('Column Type'),
required=True,
choices=common.to_choices(HIVE_TYPES))
array_type = forms.ChoiceField(
required=False,
choices=common.to_choices(HIVE_PRIMITIVE_TYPES),
label=_t("Array Value Type"))
map_key_type = forms.ChoiceField(
required=False,
choices=common.to_choices(HIVE_PRIMITIVE_TYPES),
help_text=_t("Specify if column_type is map."))
map_value_type = forms.ChoiceField(
required=False,
choices=common.to_choices(HIVE_PRIMITIVE_TYPES),
help_text=_t("Specify if column_type is map."))
class ColumnTypeForm(DependencyAwareForm):
"""
Form used to specify a column during table creation
"""
dependencies = [
("column_type", "array", "array_type"),
("column_type", "map", "map_key_type"),
("column_type", "map", "map_value_type"),
("column_type", "char", "char_length"),
("column_type", "varchar", "varchar_length")
]
column_name = common.HiveIdentifierField(label=_t('Column Name'), required=True)
column_type = forms.ChoiceField(label=_t('Column Type'), required=True,
choices=common.to_choices(HIVE_TYPES))
array_type = forms.ChoiceField(required=False,
choices=common.to_choices(HIVE_PRIMITIVE_TYPES), label=_t("Array Value Type"))
map_key_type = forms.ChoiceField(required=False,
choices=common.to_choices(HIVE_PRIMITIVE_TYPES),
help_text=_t("Specify if column_type is map."))
map_value_type = forms.ChoiceField(required=False,
choices=common.to_choices(HIVE_PRIMITIVE_TYPES),
help_text=_t("Specify if column_type is map."))
char_length = forms.IntegerField(required=False, initial=1,
widget=NumberInput(attrs={'min': 1, 'max': 255}),
validators=[MinValueValidator(1), MaxValueValidator(255)],
help_text=_t("Specify if column_type is char"))
varchar_length = forms.IntegerField(required=False, initial=1,
widget=NumberInput(attrs={'min': 1, 'max': 65355}),
validators=[MinValueValidator(1), MaxValueValidator(65355)],
help_text=_t("Specify if column_is varchar"))
class PartitionTypeForm(forms.Form):
dependencies = [("column_type", "char", "char_length"),
("column_type", "varchar", "varchar_length")]
column_name = common.HiveIdentifierField(required=True)
column_type = forms.ChoiceField(
required=True, choices=common.to_choices(HIVE_PRIMITIVE_TYPES))
char_length = forms.IntegerField(
required=False,
initial=1,
widget=NumberInput(attrs={
'min': 1,
'max': 255
}),
validators=[MinValueValidator(1),
MaxValueValidator(255)],
help_text=_t("Specify if column_type is char"))
varchar_length = forms.IntegerField(
required=False,
initial=1,
widget=NumberInput(attrs={
'min': 1,
'max': 65355
}),
validators=[MinValueValidator(1),
MaxValueValidator(65355)],
help_text=_t("Specify if column_is varchar"))
class SaveResultsForm(DependencyAwareForm):
"""Used for saving the query result data"""
SAVE_TYPES = (SAVE_TYPE_TBL, SAVE_TYPE_DIR) = ('to a new table', 'to HDFS directory')
save_target = forms.ChoiceField(required=True,
choices=common.to_choices(SAVE_TYPES),
widget=forms.RadioSelect)
target_table = common.HiveIdentifierField(
label="Table Name",
required=False,
help_text="Name of the new table")
target_dir = filebrowser.forms.PathField(
label="Results Location",
required=False,
help_text="Empty directory in HDFS to put the results")
dependencies = [
('save_target', SAVE_TYPE_TBL, 'target_table'),
('save_target', SAVE_TYPE_DIR, 'target_dir'),
]
def clean_target_table(self):
tbl = self.cleaned_data.get('target_table')
if tbl:
try:
db_utils.meta_client().get_table("default", tbl)
raise forms.ValidationError('Table already exists')
except hive_metastore.ttypes.NoSuchObjectException:
pass
return tbl
class SaveResultsForm(DependencyAwareForm):
"""Used for saving the query result data"""
SAVE_TYPES = (SAVE_TYPE_TBL, SAVE_TYPE_DIR) = (_('to a new table'),
_('to HDFS directory'))
save_target = forms.ChoiceField(required=True,
choices=common.to_choices(SAVE_TYPES),
widget=forms.RadioSelect)
target_table = common.HiveIdentifierField(
label=_t("Table Name"),
required=False,
help_text=_t("Name of the new table"))
target_dir = PathField(
label=_t("Results Location"),
required=False,
help_text=_t("Empty directory in HDFS to put the results"))
dependencies = [
('save_target', SAVE_TYPE_TBL, 'target_table'),
('save_target', SAVE_TYPE_DIR, 'target_dir'),
]
def __init__(self, *args, **kwargs):
self.db = kwargs.pop('db', None)
super(SaveResultsForm, self).__init__(*args, **kwargs)
def clean_target_table(self):
tbl = self.cleaned_data.get('target_table')
if tbl:
try:
if self.db is not None:
self.db.get_table("default", tbl)
raise forms.ValidationError(_('Table already exists'))
except hive_metastore.ttypes.NoSuchObjectException:
pass
return tbl
class SaveResultsForm(DependencyAwareForm):
"""Used for saving the query result data"""
SAVE_TYPES = (SAVE_TYPE_TBL, SAVE_TYPE_DIR) = ('to a new table',
'to HDFS directory')
save_target = forms.ChoiceField(required=True,
choices=common.to_choices(SAVE_TYPES),
widget=forms.RadioSelect,
initial=SAVE_TYPE_TBL)
target_table = common.HiveIdentifierField(
label=_t("Table Name"),
required=False,
help_text=_t("Name of the new table"))
target_dir = PathField(
label=_t("Results Location"),
required=False,
help_text=_t("Empty directory in HDFS to store results."))
dependencies = [
('save_target', SAVE_TYPE_TBL, 'target_table'),
('save_target', SAVE_TYPE_DIR, 'target_dir'),
]
def __init__(self, *args, **kwargs):
self.db = kwargs.pop('db', None)
self.fs = kwargs.pop('fs', None)
super(SaveResultsForm, self).__init__(*args, **kwargs)
def clean(self):
cleaned_data = super(SaveResultsForm, self).clean()
if cleaned_data.get('save_target') == SaveResultsForm.SAVE_TYPE_TBL:
tbl = cleaned_data.get('target_table')
if tbl:
try:
if self.db is not None:
self.db.get_table('default',
tbl) # Assumes 'default' DB
self._errors['target_table'] = self.error_class(
[_('Table already exists')])
del cleaned_data['target_table']
except Exception:
pass
elif cleaned_data['save_target'] == SaveResultsForm.SAVE_TYPE_DIR:
target_dir = cleaned_data['target_dir']
if not target_dir.startswith('/'):
self._errors['target_dir'] = self.error_class(
[_('Directory should start with /')])
elif self.fs.exists(target_dir):
self._errors['target_dir'] = self.error_class([
_('Directory already exists.')
]) # Overwrite destination directory content
return cleaned_data
class CreateTableForm(DependencyAwareForm):
"""
Form used in the create table page
"""
dependencies = []
# Basic Data
name = common.HiveIdentifierField(label=_t("Table Name"), required=True)
comment = forms.CharField(label=_t("Description"), required=False)
# Row Formatting
row_format = forms.ChoiceField(required=True,
choices=common.to_choices([ "Delimited", "SerDe" ]),
initial="Delimited")
# Delimited Row
# These initials are per LazySimpleSerDe.DefaultSeparators
field_terminator = ChoiceOrOtherField(label=_t("Field terminator"), required=False, initial=TERMINATOR_CHOICES[0][0],
choices=TERMINATOR_CHOICES)
collection_terminator = ChoiceOrOtherField(label=_t("Collection terminator"), required=False, initial=TERMINATOR_CHOICES[1][0],
choices=TERMINATOR_CHOICES)
map_key_terminator = ChoiceOrOtherField(label=_t("Map key terminator"), required=False, initial=TERMINATOR_CHOICES[2][0],
choices=TERMINATOR_CHOICES)
dependencies += [
("row_format", "Delimited", "field_terminator"),
("row_format", "Delimited", "collection_terminator"),
("row_format", "Delimited", "map_key_terminator"),
]
# Serde Row
serde_name = forms.CharField(required=False, label=_t("SerDe Name"))
serde_properties = forms.CharField(
required=False,
help_text=_t("Comma-separated list of key-value pairs. E.g. 'p1=v1, p2=v2'"))
dependencies += [
("row_format", "SerDe", "serde_name"),
("row_format", "SerDe", "serde_properties"),
]
# File Format
file_format = forms.ChoiceField(required=False, initial="TextFile",
choices=common.to_choices(["TextFile", "SequenceFile", "InputFormat"]),
widget=forms.RadioSelect)
input_format_class = forms.CharField(required=False, label=_t("InputFormat Class"))
output_format_class = forms.CharField(required=False, label=_t("OutputFormat Class"))
dependencies += [
("file_format", "InputFormat", "input_format_class"),
("file_format", "InputFormat", "output_format_class"),
]
# External?
use_default_location = forms.BooleanField(required=False, initial=True, label=_t("Use default location."))
external_location = forms.CharField(required=False, help_text=_t("Path to HDFS directory or file of table data."))
dependencies += [
("use_default_location", False, "external_location")
]
def clean_field_terminator(self):
return _clean_terminator(self.cleaned_data.get('field_terminator'))
def clean_collection_terminator(self):
return _clean_terminator(self.cleaned_data.get('collection_terminator'))
def clean_map_key_terminator(self):
return _clean_terminator(self.cleaned_data.get('map_key_terminator'))
def clean_name(self):
return _clean_tablename(self.db, self.cleaned_data['name'], self.database)
class PartitionTypeForm(forms.Form):
column_name = common.HiveIdentifierField(required=True)
column_type = forms.ChoiceField(
required=True, choices=common.to_choices(HIVE_PRIMITIVE_TYPES))
class ReportConditionBoolForm(forms.Form):
bool = forms.ChoiceField(label='And/Or',
required=True,
choices=common.to_choices(['AND', 'OR']))
class ReportConditionForm(forms.Form):
l_source = forms.ChoiceField(label='Source',
initial='table',
choices=common.to_choices(
common.SELECTION_SOURCE))
l_table = forms.CharField(label='Table name/alias', required=False)
l_col = forms.CharField(label='Column name', required=False)
l_constant = forms.CharField(label='Constant', required=False)
op = forms.ChoiceField(label='Condition',
choices=common.to_choices(common.RELATION_OPS))
r_source = forms.ChoiceField(label='Source',
required=False,
initial='table',
choices=common.to_choices(
common.SELECTION_SOURCE))
r_table = forms.CharField(label='Table name/alias', required=False)
r_col = forms.CharField(label='Column name', required=False)
r_constant = forms.CharField(label='Constant', required=False)
def clean(self):
if self.errors:
return
# Verify unary operators constraints
check_right = True
op = self.cleaned_data['op']
if op in common.RELATION_OPS_UNARY:
if self.cleaned_data.get('r_source') or self.cleaned_data.get(
'r_cond'):
raise forms.ValidationError(
'Operator %s does not take the right operand' % (op, ))
check_right = False
else:
if not self.cleaned_data.get(
'l_source') or not self.cleaned_data.get('r_source'):
raise forms.ValidationError('Operator %s takes both operands' %
(op, ))
# Verify the lhs values match the source
l_source = self.cleaned_data['l_source']
l_constant = self.cleaned_data.get('l_constant')
_field_source_check(l_source,
'Constant (Left)',
l_constant,
is_from_table=False)
l_table = self.cleaned_data.get('l_table')
_field_source_check(l_source,
'Table (Left)',
l_table,
is_from_table=True)
l_col = self.cleaned_data.get('l_col')
_field_source_check(l_source,
'Column (Left)',
l_col,
is_from_table=True)
if check_right:
# Verify the rhs values match the source
r_source = self.cleaned_data['r_source']
r_constant = self.cleaned_data.get('r_constant')
_field_source_check(r_source,
'Constant (Right)',
r_constant,
is_from_table=False)
r_table = self.cleaned_data.get('r_table')
_field_source_check(r_source,
'Table (Right)',
r_table,
is_from_table=True)
r_col = self.cleaned_data.get('r_col')
_field_source_check(r_source,
'Column (Right)',
r_col,
is_from_table=True)
return self.cleaned_data
def get_boolean_condition(self, table_alias_dict):
if not self.is_valid():
assert False, 'ReportConditionForm is not valid'
return None
op = self.cleaned_data['op']
lhs = self._make_selection(table_alias_dict, is_left=True)
if op in common.RELATION_OPS_UNARY:
return report_gen.BooleanCondition(lhs, op)
rhs = self._make_selection(table_alias_dict, is_left=False)
return report_gen.BooleanCondition(lhs, op, rhs)
def _make_selection(self, table_alias_dict, is_left):
if is_left:
prefix = 'l_'
else:
prefix = 'r_'
source = self.cleaned_data[prefix + 'source']
if source == 'table':
table = self.cleaned_data[prefix + 'table']
col = self.cleaned_data[prefix + 'col']
try:
return report_gen.ColumnSelection(table_alias_dict[table], col)
except KeyError:
raise forms.ValidationError('Unknown table "%s" in condition' %
(table, ))
constant = self.cleaned_data[prefix + 'constant']
return report_gen.ConstSelection(constant)
class ReportColumnForm(forms.Form):
"""
A form representing a column in the report.
"""
# If not 'display', then source must be 'table'
display = forms.BooleanField(label='Display', required=False, initial=True)
# Shown iff 'display'. 'source' is not required, but will be set during clean
source = forms.ChoiceField(label='Source',
required=False,
initial='table',
choices=common.to_choices(
common.SELECTION_SOURCE))
# Shown iff 'display'
agg = forms.ChoiceField(label='Aggregate',
required=False,
choices=common.to_choices(common.AGGREGATIONS))
# Shown iff 'display'
distinct = forms.BooleanField(label="Distinct", required=False)
# Shown iff 'source' is 'constant'
constant = forms.CharField(label='Constant value', required=False)
# Shown iff 'source' is 'table'
table_alias = common.HiveIdentifierField(label='Table alias',
required=False)
# Shown iff 'source' is 'table'
col = forms.CharField(label='From column', required=False)
# Shown iff 'display', and 'source' is 'table'
col_alias = common.HiveIdentifierField(label='Column alias',
required=False)
# Shown iff 'display', and 'source' is 'table'
sort = forms.ChoiceField(label='Sort',
required=False,
choices=common.to_choices(common.SORT_OPTIONS))
# Shown iff 'sort'
sort_order = forms.IntegerField(label='Sort order',
required=False,
min_value=1)
# Shown iff 'display', and 'source' is 'table'
group_order = forms.IntegerField(label='Group order',
required=False,
min_value=1)
def __init__(self, *args, **kwargs):
forms.Form.__init__(self, *args, **kwargs)
# Shown iff 'source' is 'table'
self.fields['table'] = common.HiveTableChoiceField(label='From table',
required=False)
def _display_check(self):
"""Reconcile 'display' with 'source'"""
src = self.cleaned_data.get('source')
if not self.cleaned_data.get('display'):
if src and src != 'table':
raise forms.ValidationError(
'Source must be "table" when not displaying column')
self.cleaned_data['source'] = 'table'
if self.cleaned_data.get('col_alias'):
raise forms.ValidationError(
'Column alias not applicable when not displaying column')
else:
if not src:
raise forms.ValidationError('Source value missing')
def clean_display(self):
"""Make sure display is set"""
return self.cleaned_data.get('display', False)
def clean_sort(self):
"""Set sort_hql accordingly"""
dir = self.cleaned_data.get('sort')
if dir == 'ascending':
self.cleaned_data['sort_hql'] = 'ASC'
elif dir == 'descending':
self.cleaned_data['sort_hql'] = 'DESC'
elif self.cleaned_data.has_key('sort_hql'):
del self.cleaned_data['sort_hql']
return dir
def clean(self):
self.qtable = None
self.selection = None
self._display_check()
if self.cleaned_data.get('sort') and not self.cleaned_data['sort_hql']:
raise KeyError()
# Verify that the 'source' field is consistent with the other fields
source = self.cleaned_data.get('source')
if not source:
return None # No point since we can't get source
constant_val = self.cleaned_data.get('constant')
_field_source_check(source,
'Constant',
constant_val,
is_from_table=False)
table_val = self.cleaned_data.get('table')
_field_source_check(source,
'From table',
table_val,
is_from_table=True)
col_val = self.cleaned_data.get('col')
_field_source_check(source, 'From column', col_val, is_from_table=True)
if self.cleaned_data.get(
'sort', '') and not self.cleaned_data.get('sort_order', ''):
raise forms.ValidationError('Sort order missing')
if table_val:
# Column must belong to the table
self.qtable = report_gen.QTable(
table_val, self.cleaned_data.get('table_alias'))
if col_val == '*':
if self.cleaned_data.get('col_alias'):
raise forms.ValidationError(
'Alias not applicable for selecting "*"')
elif col_val not in self.qtable.get_columns():
raise forms.ValidationError('Invalid column name "%s"' %
(col_val, ))
# ColumnSelection object
self.selection = report_gen.ColumnSelection(
self.qtable, col_val, self.cleaned_data.get('col_alias'))
else:
# ConstSelection object
self.selection = report_gen.ConstSelection(
constant_val, self.cleaned_data.get('col_alias'))
self.selection.distinct = self.cleaned_data.get('distinct', False)
self.selection.set_aggregation(self.cleaned_data.get('agg', ''))
if self.errors:
delattr(self, 'selection')
return self.cleaned_data
