class ColumnTypeForm(DependencyAwareForm): """ Form used to specify a column during table creation """ dependencies = [ ("column_type", "array", "array_type"), ("column_type", "map", "map_key_type"), ("column_type", "map", "map_value_type"), ] column_name = common.HiveIdentifierField(label=_t('Column Name'), required=True) column_type = forms.ChoiceField(label=_t('Column Type'), required=True, choices=common.to_choices(HIVE_TYPES)) array_type = forms.ChoiceField( required=False, choices=common.to_choices(HIVE_PRIMITIVE_TYPES), label=_t("Array Value Type")) map_key_type = forms.ChoiceField( required=False, choices=common.to_choices(HIVE_PRIMITIVE_TYPES), help_text=_t("Specify if column_type is map.")) map_value_type = forms.ChoiceField( required=False, choices=common.to_choices(HIVE_PRIMITIVE_TYPES), help_text=_t("Specify if column_type is map."))
class ColumnTypeForm(DependencyAwareForm): """ Form used to specify a column during table creation """ dependencies = [ ("column_type", "array", "array_type"), ("column_type", "map", "map_key_type"), ("column_type", "map", "map_value_type"), ("column_type", "char", "char_length"), ("column_type", "varchar", "varchar_length") ] column_name = common.HiveIdentifierField(label=_t('Column Name'), required=True) column_type = forms.ChoiceField(label=_t('Column Type'), required=True, choices=common.to_choices(HIVE_TYPES)) array_type = forms.ChoiceField(required=False, choices=common.to_choices(HIVE_PRIMITIVE_TYPES), label=_t("Array Value Type")) map_key_type = forms.ChoiceField(required=False, choices=common.to_choices(HIVE_PRIMITIVE_TYPES), help_text=_t("Specify if column_type is map.")) map_value_type = forms.ChoiceField(required=False, choices=common.to_choices(HIVE_PRIMITIVE_TYPES), help_text=_t("Specify if column_type is map.")) char_length = forms.IntegerField(required=False, initial=1, widget=NumberInput(attrs={'min': 1, 'max': 255}), validators=[MinValueValidator(1), MaxValueValidator(255)], help_text=_t("Specify if column_type is char")) varchar_length = forms.IntegerField(required=False, initial=1, widget=NumberInput(attrs={'min': 1, 'max': 65355}), validators=[MinValueValidator(1), MaxValueValidator(65355)], help_text=_t("Specify if column_is varchar"))
class PartitionTypeForm(forms.Form): dependencies = [("column_type", "char", "char_length"), ("column_type", "varchar", "varchar_length")] column_name = common.HiveIdentifierField(required=True) column_type = forms.ChoiceField( required=True, choices=common.to_choices(HIVE_PRIMITIVE_TYPES)) char_length = forms.IntegerField( required=False, initial=1, widget=NumberInput(attrs={ 'min': 1, 'max': 255 }), validators=[MinValueValidator(1), MaxValueValidator(255)], help_text=_t("Specify if column_type is char")) varchar_length = forms.IntegerField( required=False, initial=1, widget=NumberInput(attrs={ 'min': 1, 'max': 65355 }), validators=[MinValueValidator(1), MaxValueValidator(65355)], help_text=_t("Specify if column_is varchar"))
class SaveResultsForm(DependencyAwareForm): """Used for saving the query result data""" SAVE_TYPES = (SAVE_TYPE_TBL, SAVE_TYPE_DIR) = ('to a new table', 'to HDFS directory') save_target = forms.ChoiceField(required=True, choices=common.to_choices(SAVE_TYPES), widget=forms.RadioSelect) target_table = common.HiveIdentifierField( label="Table Name", required=False, help_text="Name of the new table") target_dir = filebrowser.forms.PathField( label="Results Location", required=False, help_text="Empty directory in HDFS to put the results") dependencies = [ ('save_target', SAVE_TYPE_TBL, 'target_table'), ('save_target', SAVE_TYPE_DIR, 'target_dir'), ] def clean_target_table(self): tbl = self.cleaned_data.get('target_table') if tbl: try: db_utils.meta_client().get_table("default", tbl) raise forms.ValidationError('Table already exists') except hive_metastore.ttypes.NoSuchObjectException: pass return tbl
class SaveResultsForm(DependencyAwareForm): """Used for saving the query result data""" SAVE_TYPES = (SAVE_TYPE_TBL, SAVE_TYPE_DIR) = (_('to a new table'), _('to HDFS directory')) save_target = forms.ChoiceField(required=True, choices=common.to_choices(SAVE_TYPES), widget=forms.RadioSelect) target_table = common.HiveIdentifierField( label=_t("Table Name"), required=False, help_text=_t("Name of the new table")) target_dir = PathField( label=_t("Results Location"), required=False, help_text=_t("Empty directory in HDFS to put the results")) dependencies = [ ('save_target', SAVE_TYPE_TBL, 'target_table'), ('save_target', SAVE_TYPE_DIR, 'target_dir'), ] def __init__(self, *args, **kwargs): self.db = kwargs.pop('db', None) super(SaveResultsForm, self).__init__(*args, **kwargs) def clean_target_table(self): tbl = self.cleaned_data.get('target_table') if tbl: try: if self.db is not None: self.db.get_table("default", tbl) raise forms.ValidationError(_('Table already exists')) except hive_metastore.ttypes.NoSuchObjectException: pass return tbl
class SaveResultsForm(DependencyAwareForm): """Used for saving the query result data""" SAVE_TYPES = (SAVE_TYPE_TBL, SAVE_TYPE_DIR) = ('to a new table', 'to HDFS directory') save_target = forms.ChoiceField(required=True, choices=common.to_choices(SAVE_TYPES), widget=forms.RadioSelect, initial=SAVE_TYPE_TBL) target_table = common.HiveIdentifierField( label=_t("Table Name"), required=False, help_text=_t("Name of the new table")) target_dir = PathField( label=_t("Results Location"), required=False, help_text=_t("Empty directory in HDFS to store results.")) dependencies = [ ('save_target', SAVE_TYPE_TBL, 'target_table'), ('save_target', SAVE_TYPE_DIR, 'target_dir'), ] def __init__(self, *args, **kwargs): self.db = kwargs.pop('db', None) self.fs = kwargs.pop('fs', None) super(SaveResultsForm, self).__init__(*args, **kwargs) def clean(self): cleaned_data = super(SaveResultsForm, self).clean() if cleaned_data.get('save_target') == SaveResultsForm.SAVE_TYPE_TBL: tbl = cleaned_data.get('target_table') if tbl: try: if self.db is not None: self.db.get_table('default', tbl) # Assumes 'default' DB self._errors['target_table'] = self.error_class( [_('Table already exists')]) del cleaned_data['target_table'] except Exception: pass elif cleaned_data['save_target'] == SaveResultsForm.SAVE_TYPE_DIR: target_dir = cleaned_data['target_dir'] if not target_dir.startswith('/'): self._errors['target_dir'] = self.error_class( [_('Directory should start with /')]) elif self.fs.exists(target_dir): self._errors['target_dir'] = self.error_class([ _('Directory already exists.') ]) # Overwrite destination directory content return cleaned_data
class CreateTableForm(DependencyAwareForm): """ Form used in the create table page """ dependencies = [] # Basic Data name = common.HiveIdentifierField(label=_t("Table Name"), required=True) comment = forms.CharField(label=_t("Description"), required=False) # Row Formatting row_format = forms.ChoiceField(required=True, choices=common.to_choices([ "Delimited", "SerDe" ]), initial="Delimited") # Delimited Row # These initials are per LazySimpleSerDe.DefaultSeparators field_terminator = ChoiceOrOtherField(label=_t("Field terminator"), required=False, initial=TERMINATOR_CHOICES[0][0], choices=TERMINATOR_CHOICES) collection_terminator = ChoiceOrOtherField(label=_t("Collection terminator"), required=False, initial=TERMINATOR_CHOICES[1][0], choices=TERMINATOR_CHOICES) map_key_terminator = ChoiceOrOtherField(label=_t("Map key terminator"), required=False, initial=TERMINATOR_CHOICES[2][0], choices=TERMINATOR_CHOICES) dependencies += [ ("row_format", "Delimited", "field_terminator"), ("row_format", "Delimited", "collection_terminator"), ("row_format", "Delimited", "map_key_terminator"), ] # Serde Row serde_name = forms.CharField(required=False, label=_t("SerDe Name")) serde_properties = forms.CharField( required=False, help_text=_t("Comma-separated list of key-value pairs. E.g. 'p1=v1, p2=v2'")) dependencies += [ ("row_format", "SerDe", "serde_name"), ("row_format", "SerDe", "serde_properties"), ] # File Format file_format = forms.ChoiceField(required=False, initial="TextFile", choices=common.to_choices(["TextFile", "SequenceFile", "InputFormat"]), widget=forms.RadioSelect) input_format_class = forms.CharField(required=False, label=_t("InputFormat Class")) output_format_class = forms.CharField(required=False, label=_t("OutputFormat Class")) dependencies += [ ("file_format", "InputFormat", "input_format_class"), ("file_format", "InputFormat", "output_format_class"), ] # External? use_default_location = forms.BooleanField(required=False, initial=True, label=_t("Use default location.")) external_location = forms.CharField(required=False, help_text=_t("Path to HDFS directory or file of table data.")) dependencies += [ ("use_default_location", False, "external_location") ] def clean_field_terminator(self): return _clean_terminator(self.cleaned_data.get('field_terminator')) def clean_collection_terminator(self): return _clean_terminator(self.cleaned_data.get('collection_terminator')) def clean_map_key_terminator(self): return _clean_terminator(self.cleaned_data.get('map_key_terminator')) def clean_name(self): return _clean_tablename(self.db, self.cleaned_data['name'], self.database)
class PartitionTypeForm(forms.Form): column_name = common.HiveIdentifierField(required=True) column_type = forms.ChoiceField( required=True, choices=common.to_choices(HIVE_PRIMITIVE_TYPES))
class ReportConditionBoolForm(forms.Form): bool = forms.ChoiceField(label='And/Or', required=True, choices=common.to_choices(['AND', 'OR']))
class ReportConditionForm(forms.Form): l_source = forms.ChoiceField(label='Source', initial='table', choices=common.to_choices( common.SELECTION_SOURCE)) l_table = forms.CharField(label='Table name/alias', required=False) l_col = forms.CharField(label='Column name', required=False) l_constant = forms.CharField(label='Constant', required=False) op = forms.ChoiceField(label='Condition', choices=common.to_choices(common.RELATION_OPS)) r_source = forms.ChoiceField(label='Source', required=False, initial='table', choices=common.to_choices( common.SELECTION_SOURCE)) r_table = forms.CharField(label='Table name/alias', required=False) r_col = forms.CharField(label='Column name', required=False) r_constant = forms.CharField(label='Constant', required=False) def clean(self): if self.errors: return # Verify unary operators constraints check_right = True op = self.cleaned_data['op'] if op in common.RELATION_OPS_UNARY: if self.cleaned_data.get('r_source') or self.cleaned_data.get( 'r_cond'): raise forms.ValidationError( 'Operator %s does not take the right operand' % (op, )) check_right = False else: if not self.cleaned_data.get( 'l_source') or not self.cleaned_data.get('r_source'): raise forms.ValidationError('Operator %s takes both operands' % (op, )) # Verify the lhs values match the source l_source = self.cleaned_data['l_source'] l_constant = self.cleaned_data.get('l_constant') _field_source_check(l_source, 'Constant (Left)', l_constant, is_from_table=False) l_table = self.cleaned_data.get('l_table') _field_source_check(l_source, 'Table (Left)', l_table, is_from_table=True) l_col = self.cleaned_data.get('l_col') _field_source_check(l_source, 'Column (Left)', l_col, is_from_table=True) if check_right: # Verify the rhs values match the source r_source = self.cleaned_data['r_source'] r_constant = self.cleaned_data.get('r_constant') _field_source_check(r_source, 'Constant (Right)', r_constant, is_from_table=False) r_table = self.cleaned_data.get('r_table') _field_source_check(r_source, 'Table (Right)', r_table, is_from_table=True) r_col = self.cleaned_data.get('r_col') _field_source_check(r_source, 'Column (Right)', r_col, is_from_table=True) return self.cleaned_data def get_boolean_condition(self, table_alias_dict): if not self.is_valid(): assert False, 'ReportConditionForm is not valid' return None op = self.cleaned_data['op'] lhs = self._make_selection(table_alias_dict, is_left=True) if op in common.RELATION_OPS_UNARY: return report_gen.BooleanCondition(lhs, op) rhs = self._make_selection(table_alias_dict, is_left=False) return report_gen.BooleanCondition(lhs, op, rhs) def _make_selection(self, table_alias_dict, is_left): if is_left: prefix = 'l_' else: prefix = 'r_' source = self.cleaned_data[prefix + 'source'] if source == 'table': table = self.cleaned_data[prefix + 'table'] col = self.cleaned_data[prefix + 'col'] try: return report_gen.ColumnSelection(table_alias_dict[table], col) except KeyError: raise forms.ValidationError('Unknown table "%s" in condition' % (table, )) constant = self.cleaned_data[prefix + 'constant'] return report_gen.ConstSelection(constant)
class ReportColumnForm(forms.Form): """ A form representing a column in the report. """ # If not 'display', then source must be 'table' display = forms.BooleanField(label='Display', required=False, initial=True) # Shown iff 'display'. 'source' is not required, but will be set during clean source = forms.ChoiceField(label='Source', required=False, initial='table', choices=common.to_choices( common.SELECTION_SOURCE)) # Shown iff 'display' agg = forms.ChoiceField(label='Aggregate', required=False, choices=common.to_choices(common.AGGREGATIONS)) # Shown iff 'display' distinct = forms.BooleanField(label="Distinct", required=False) # Shown iff 'source' is 'constant' constant = forms.CharField(label='Constant value', required=False) # Shown iff 'source' is 'table' table_alias = common.HiveIdentifierField(label='Table alias', required=False) # Shown iff 'source' is 'table' col = forms.CharField(label='From column', required=False) # Shown iff 'display', and 'source' is 'table' col_alias = common.HiveIdentifierField(label='Column alias', required=False) # Shown iff 'display', and 'source' is 'table' sort = forms.ChoiceField(label='Sort', required=False, choices=common.to_choices(common.SORT_OPTIONS)) # Shown iff 'sort' sort_order = forms.IntegerField(label='Sort order', required=False, min_value=1) # Shown iff 'display', and 'source' is 'table' group_order = forms.IntegerField(label='Group order', required=False, min_value=1) def __init__(self, *args, **kwargs): forms.Form.__init__(self, *args, **kwargs) # Shown iff 'source' is 'table' self.fields['table'] = common.HiveTableChoiceField(label='From table', required=False) def _display_check(self): """Reconcile 'display' with 'source'""" src = self.cleaned_data.get('source') if not self.cleaned_data.get('display'): if src and src != 'table': raise forms.ValidationError( 'Source must be "table" when not displaying column') self.cleaned_data['source'] = 'table' if self.cleaned_data.get('col_alias'): raise forms.ValidationError( 'Column alias not applicable when not displaying column') else: if not src: raise forms.ValidationError('Source value missing') def clean_display(self): """Make sure display is set""" return self.cleaned_data.get('display', False) def clean_sort(self): """Set sort_hql accordingly""" dir = self.cleaned_data.get('sort') if dir == 'ascending': self.cleaned_data['sort_hql'] = 'ASC' elif dir == 'descending': self.cleaned_data['sort_hql'] = 'DESC' elif self.cleaned_data.has_key('sort_hql'): del self.cleaned_data['sort_hql'] return dir def clean(self): self.qtable = None self.selection = None self._display_check() if self.cleaned_data.get('sort') and not self.cleaned_data['sort_hql']: raise KeyError() # Verify that the 'source' field is consistent with the other fields source = self.cleaned_data.get('source') if not source: return None # No point since we can't get source constant_val = self.cleaned_data.get('constant') _field_source_check(source, 'Constant', constant_val, is_from_table=False) table_val = self.cleaned_data.get('table') _field_source_check(source, 'From table', table_val, is_from_table=True) col_val = self.cleaned_data.get('col') _field_source_check(source, 'From column', col_val, is_from_table=True) if self.cleaned_data.get( 'sort', '') and not self.cleaned_data.get('sort_order', ''): raise forms.ValidationError('Sort order missing') if table_val: # Column must belong to the table self.qtable = report_gen.QTable( table_val, self.cleaned_data.get('table_alias')) if col_val == '*': if self.cleaned_data.get('col_alias'): raise forms.ValidationError( 'Alias not applicable for selecting "*"') elif col_val not in self.qtable.get_columns(): raise forms.ValidationError('Invalid column name "%s"' % (col_val, )) # ColumnSelection object self.selection = report_gen.ColumnSelection( self.qtable, col_val, self.cleaned_data.get('col_alias')) else: # ConstSelection object self.selection = report_gen.ConstSelection( constant_val, self.cleaned_data.get('col_alias')) self.selection.distinct = self.cleaned_data.get('distinct', False) self.selection.set_aggregation(self.cleaned_data.get('agg', '')) if self.errors: delattr(self, 'selection') return self.cleaned_data