Пример #1
0
 def _find_table_name(self, data):
     if not self.table_name:
         if pymongo and isinstance(data, pymongo.collection.Collection):
             self.table_name = data.name
         elif hasattr(data, 'lower'):  # duck-type string test
             if os.path.isfile(data):
                 (file_path, file_extension) = os.path.splitext(data)
                 self.table_name = os.path.split(file_path)[1].lower()
     self.table_name = self.table_name or \
                       'generated_table%s' % Table.table_index
     self.table_name = reshape.clean_key_name(self.table_name)
     Table.table_index += 1
Пример #2
0
 def _find_table_name(self, data):
     if not self.table_name:
         if pymongo and isinstance(data, pymongo.collection.Collection):
             self.table_name = data.name
         elif hasattr(data, 'lower'):  # duck-type string test
             if os.path.isfile(data):
                 (file_path, file_extension) = os.path.splitext(data)
                 self.table_name = os.path.split(file_path)[1]
     self.table_name = self.table_name or \
                       'generated_table%s' % Table.table_index
     self.table_name = reshape.clean_key_name(self.table_name)
     Table.table_index += 1
Пример #3
0
    def __init__(self, data, table_name=None, default_dialect=None, 
                 save_metadata_to=None, metadata_source=None,
                 varying_length_text = False, uniques=False, 
                 pk_name=None, force_pk=False,
                 _parent_table=None, _fk_field_name=None, reorder=False,
                 loglevel=logging.WARN):
        """
        Initialize a Table and load its data.
        
        If ``varying_length_text`` is ``True``, text columns will be TEXT rather than VARCHAR.
        This *improves* performance in PostgreSQL.
        
        If a ``metadata<timestamp>`` YAML file generated from a previous ddlgenerator run is
        provided, *only* ``INSERT`` statements will be produced, and the table structure 
        determined during the previous run will be assumed.
        """
        logging.getLogger().setLevel(loglevel) 
        self.table_name = None
        self.varying_length_text = varying_length_text
        self._load_data(data)
        if hasattr(self.data, 'lower'):
            raise SyntaxError("Data was interpreted as a single string - no table structure:\n%s" 
                              % self.data[:100])
        self.table_name = table_name or self.table_name or 'generated_table%s' % Table.table_index
        Table.table_index += 1
        self.table_name = reshape.clean_key_name(self.table_name)
        
        if not hasattr(self.data, 'append') and not hasattr(self.data, '__next__') \
            and not hasattr(self.data, 'next'):
            self.data = [self.data,]
        self.data = reshape.walk_and_clean(self.data)
        
        (self.data, self.pk_name, children, child_fk_names) = reshape.unnest_children(
            data=self.data, parent_name=self.table_name, pk_name=pk_name, force_pk=force_pk)
      
        self.default_dialect = default_dialect
        self.comments = {}
        child_metadata_sources = {}
        if metadata_source:
            if isinstance(metadata_source, OrderedDict):
                logging.info('Column metadata passed in as OrderedDict')
                self.columns = metadata_source
            else:
                logging.info('Pulling column metadata from file %s' % metadata_source)
                with open(metadata_source) as infile:
                    self.columns = yaml.load(infile.read())
            for (col_name, col) in self.columns.items():
                if isinstance(col, OrderedDict):
                    child_metadata_sources[col_name] = col
                    self.columns.pop(col_name)
                else:
                    self._fill_metadata_from_sample(col)
        else:
            self._determine_types(varying_length_text, uniques=uniques)

        if reorder:
            ordered_columns = OrderedDict()
            if pk_name and pk_name in self.columns:
                ordered_columns[pk_name] = self.columns.pop(pk_name)
            for (c, v) in sorted(self.columns.items()):
                ordered_columns[c] = v
            self.columns = ordered_columns    
            
        if _parent_table:
            fk = sa.ForeignKey('%s.%s' % (_parent_table.table_name, _parent_table.pk_name))
        else:
            fk = None
            
        column_args = []
        self.table = sa.Table(self.table_name, metadata, 
                              *[sa.Column(cname, col['satype'], 
                                          fk if fk and (_fk_field_name == cname) else None,
                                          primary_key=(cname == self.pk_name),
                                          unique=col['is_unique'],
                                          nullable=col['is_nullable'],
                                          doc=self.comments.get(cname)) 
                                for (cname, col) in self.columns.items()
                                if True
                                ])
      
        self.children = {child_name: Table(child_data, table_name=child_name, 
                                           default_dialect=self.default_dialect, 
                                           varying_length_text = varying_length_text, 
                                           uniques=uniques, pk_name=pk_name, force_pk=force_pk,
                                           _parent_table=self, reorder=reorder,
                                           _fk_field_name = child_fk_names[child_name],
                                           metadata_source = child_metadata_sources.get(child_name),
                                           loglevel=loglevel)
                         for (child_name, child_data) in children.items()}
       
        if save_metadata_to:
            if not save_metadata_to.endswith(('.yml','yaml')):
                save_metadata_to += '.yaml'
            with open(save_metadata_to, 'w') as outfile:
                outfile.write(yaml.dump(self._saveable_metadata()))
            logging.info('Pass ``--save-metadata-to %s`` next time to re-use structure' %
                         save_metadata_to)
Пример #4
0
    def __init__(self,
                 data,
                 table_name=None,
                 default_dialect=None,
                 save_metadata_to=None,
                 metadata_source=None,
                 varying_length_text=False,
                 uniques=False,
                 pk_name=None,
                 force_pk=False,
                 _parent_table=None,
                 _fk_field_name=None,
                 reorder=False,
                 loglevel=logging.WARN):
        """
        Initialize a Table and load its data.
        
        If ``varying_length_text`` is ``True``, text columns will be TEXT rather than VARCHAR.
        This *improves* performance in PostgreSQL.
        
        If a ``metadata<timestamp>`` YAML file generated from a previous ddlgenerator run is
        provided, *only* ``INSERT`` statements will be produced, and the table structure 
        determined during the previous run will be assumed.
        """
        logging.getLogger().setLevel(loglevel)
        self.table_name = None
        self.varying_length_text = varying_length_text
        self._load_data(data)
        if hasattr(self.data, 'lower'):
            raise SyntaxError(
                "Data was interpreted as a single string - no table structure:\n%s"
                % self.data[:100])
        self.table_name = table_name or self.table_name or 'generated_table%s' % Table.table_index
        Table.table_index += 1
        self.table_name = reshape.clean_key_name(self.table_name)

        if not hasattr(self.data, 'append') and not hasattr(self.data, '__next__') \
            and not hasattr(self.data, 'next'):
            self.data = [
                self.data,
            ]
        self.data = reshape.walk_and_clean(self.data)

        (self.data, self.pk_name, children,
         child_fk_names) = reshape.unnest_children(data=self.data,
                                                   parent_name=self.table_name,
                                                   pk_name=pk_name,
                                                   force_pk=force_pk)

        self.default_dialect = default_dialect
        self.comments = {}
        child_metadata_sources = {}
        if metadata_source:
            if isinstance(metadata_source, OrderedDict):
                logging.info('Column metadata passed in as OrderedDict')
                self.columns = metadata_source
            else:
                logging.info('Pulling column metadata from file %s' %
                             metadata_source)
                with open(metadata_source) as infile:
                    self.columns = yaml.load(infile.read())
            for (col_name, col) in self.columns.items():
                if isinstance(col, OrderedDict):
                    child_metadata_sources[col_name] = col
                    self.columns.pop(col_name)
                else:
                    self._fill_metadata_from_sample(col)
        else:
            self._determine_types(varying_length_text, uniques=uniques)

        if reorder:
            ordered_columns = OrderedDict()
            if pk_name and pk_name in self.columns:
                ordered_columns[pk_name] = self.columns.pop(pk_name)
            for (c, v) in sorted(self.columns.items()):
                ordered_columns[c] = v
            self.columns = ordered_columns

        if _parent_table:
            fk = sa.ForeignKey(
                '%s.%s' % (_parent_table.table_name, _parent_table.pk_name))
        else:
            fk = None

        column_args = []
        self.table = sa.Table(
            self.table_name, metadata, *[
                sa.Column(cname,
                          col['satype'],
                          fk if fk and (_fk_field_name == cname) else None,
                          primary_key=(cname == self.pk_name),
                          unique=col['is_unique'],
                          nullable=col['is_nullable'],
                          doc=self.comments.get(cname))
                for (cname, col) in self.columns.items() if True
            ])

        self.children = {
            child_name:
            Table(child_data,
                  table_name=child_name,
                  default_dialect=self.default_dialect,
                  varying_length_text=varying_length_text,
                  uniques=uniques,
                  pk_name=pk_name,
                  force_pk=force_pk,
                  _parent_table=self,
                  reorder=reorder,
                  _fk_field_name=child_fk_names[child_name],
                  metadata_source=child_metadata_sources.get(child_name),
                  loglevel=loglevel)
            for (child_name, child_data) in children.items()
        }

        if save_metadata_to:
            if not save_metadata_to.endswith(('.yml', 'yaml')):
                save_metadata_to += '.yaml'
            with open(save_metadata_to, 'w') as outfile:
                outfile.write(yaml.dump(self._saveable_metadata()))
            logging.info(
                'Pass ``--save-metadata-to %s`` next time to re-use structure'
                % save_metadata_to)