def data_generator(dburl, table, excludes=None, schema='public'):
engine, table = db_connect_table(dburl, table, schema=schema)
columns = exclude_columns(table.columns, excludes)
yield [c.name for c in columns]
with scoped_session(engine) as session:
query = session.query(*columns).yield_per(1000)
for row in query:
yield row
def metadata_dict(dburl, table, extended=None, excludes=None, schema='public'):
if extended is None:
extended = {}
_, table = db_connect_table(dburl, table, schema=schema)
metadata, fields = metadata_factory(table.name)
metadata['objects'][0].update(extended)
columns = exclude_columns(table.columns, excludes)
for col in columns:
fqname = '{}.{}'.format(table.name, col.name)
field_meta = new_field(fqname, col.name)
field_meta.update(metadata_for_dbtype(col.type))
if 'fields' in extended and col.name in extended['fields']:
field_meta.update(extended['fields'][col.name])
fields.append(field_meta)
return metadata
