def get_app_meta(
apps=None,
app_filter=lambda x: x.startswith('sec_') or x.startswith('siica_'),
app_exclude_filter=None,
verbosity=0,
save=True):
apps = apps or djdb.get_app(apps)
meta = []
for app in apps:
if filter_reject(app, app_filter, app_exclude_filter):
continue
meta += [get_db_meta(app=app, verbosity=verbosity)]
if save:
try:
with open('db_meta_%s.json' % app, 'w') as fpout:
json.dump(make_serializable(meta[-1]), fpout, indent=4)
except:
print_exc()
if save:
try:
with open('db_meta_all_apps.json', 'w') as fpout:
jsonifiable_data = make_serializable(meta)
json.dump(jsonifiable_data, fpout, indent=4)
except:
print_exc()
return meta
def get_db_meta(app=DEFAULT_APP_NAME, db_alias=None, table=None, verbosity=0, column=None):
"""Return a dict of dicts containing metadata about the database tables associated with an app
TODO: allow multiple apps
>>> get_db_meta('crawler', db_alias='default', table='crawler_wikiitem') # doctest: +ELLIPSIS
OrderedDict([('WikiItem', OrderedDict([('Meta', OrderedDict([('primary_key', None), ('count', 1332), ('db_table', u'crawler_wikiitem')])), ('id', OrderedDict([('name', 'id'), ('type', ...
"""
if app and isinstance(app, basestring):
app = djdb.get_app(app)
else:
app = djdb.get_app('')
model_names = list(mc.__name__ for mc in models.get_models(app))
meta = OrderedDict()
# inspectdb uses: for table_name in connection.introspection.table_names(cursor):
for model_name in model_names:
model = djdb.get_model(model_name, app=app)
if db_alias:
model_db_alias = db_alias
else:
model_db_alias = router.db_for_read(model)
queryset = model.objects
if model_db_alias:
queryset = queryset.using(model_db_alias)
if model and table is not None and isinstance(table, basestring):
if model._meta.db_table != table:
if verbosity>1:
print 'skipped model named %s with db table names %s.' % (model_name, model._meta.db_table)
continue
elif callable(table):
if not table(model._meta.db_table):
if verbosity>1:
print 'skipped model named %s with db table names %s.' % (model_name, model._meta.db_table)
continue
count = None
try:
print 'trying to count for model %r and db_alias %r' % (model, model_db_alias)
count = queryset.count()
except DatabaseError, e:
if verbosity:
print_exc()
print "DatabaseError: Unable to count records for model '%s' (%s) because of %s." % (model.__name__, repr(model), e)
connection.close()
except:
def load_app_meta(apps=None, app_filter=['sec_', 'siica_'], app_exclude_filter=None):
apps = apps or djdb.get_app(apps)
meta = {}
for app in apps:
# if filter_reject(app, app_filter, app_exclude_filter):
# continue
if (app_filter and not any(app.startswith(af) for af in app_filter)) and (
not app_exclude_filter or not any(app.startswith(rf) for rf in app_exclude_filter)):
continue
with open('db_meta_%s.json' % app, 'r') as fpin:
m = json.load(fpin)
for table_name, table_meta in m.iteritems():
table_name = app + '.' + table_name
for field_name, field_meta in table_meta.iteritems():
meta[table_name + '.' + field_name] = field_meta
return meta
def load_app_meta(apps=None,
app_filter=['sec_', 'siica_'],
app_exclude_filter=None):
apps = apps or djdb.get_app(apps)
meta = {}
for app in apps:
# if filter_reject(app, app_filter, app_exclude_filter):
# continue
if (app_filter and not any(app.startswith(af) for af in app_filter)
) and (not app_exclude_filter
or not any(app.startswith(rf)
for rf in app_exclude_filter)):
continue
with open('db_meta_%s.json' % app, 'r') as fpin:
m = json.load(fpin)
for table_name, table_meta in m.iteritems():
table_name = app + '.' + table_name
for field_name, field_meta in table_meta.iteritems():
meta[table_name + '.' + field_name] = field_meta
return meta
def get_app_meta(apps=None, app_filter=lambda x: x.startswith('sec_') or x.startswith('siica_'), app_exclude_filter=None, verbosity=0, save=True):
apps = apps or djdb.get_app(apps)
meta = []
for app in apps:
if filter_reject(app, app_filter, app_exclude_filter):
continue
meta += [get_db_meta(app=app, verbosity=verbosity)]
if save:
try:
with open('db_meta_%s.json' % app, 'w') as fpout:
json.dump(make_serializable(meta[-1]), fpout, indent=4)
except:
print_exc()
if save:
try:
with open('db_meta_all_apps.json', 'w') as fpout:
jsonifiable_data = make_serializable(meta)
json.dump(jsonifiable_data, fpout, indent=4)
except:
print_exc()
return meta
def get_db_meta(app=DEFAULT_APP_NAME, db_alias=None, table=None, verbosity=0, column=None):
"""Return a dict of dicts containing metadata about the database tables associated with an app
TODO: allow multiple apps
>>> get_db_meta('crawler', db_alias='default', table='crawler_wikiitem') # doctest: +ELLIPSIS
OrderedDict([('WikiItem', OrderedDict([('Meta', OrderedDict([('primary_key', None), ('count', 1332), ('db_table', u'crawler_wikiitem')])), (u'id', OrderedDict([('name', u'id'), ('type', ...
"""
if verbosity:
print 'Looking for app %r.' % (app, )
if app and isinstance(app, basestring):
app = djdb.get_app(app, verbosity=verbosity)
else:
app = djdb.get_app('')
model_names = list(mc.__name__ for mc in models.get_models(app))
if verbosity:
print 'Found %d models for app %r.' % (len(model_names), app)
meta = OrderedDict()
# inspectdb uses: for table_name in connection.introspection.table_names(cursor):
for model_name in model_names:
model = djdb.get_model(model_name, app=app)
if db_alias:
model_db_alias = db_alias
else:
model_db_alias = router.db_for_read(model)
queryset = model.objects
if model_db_alias:
queryset = queryset.using(model_db_alias)
if model and table is not None and isinstance(table, basestring):
if model._meta.db_table != table:
if verbosity>1:
print 'Skipped model named %s with db table names %s.' % (model_name, model._meta.db_table)
continue
elif callable(table):
if not table(model._meta.db_table):
if verbosity>1:
print 'Skipped model named %s with db table names %s.' % (model_name, model._meta.db_table)
continue
count = None
try:
if verbosity > 1:
print 'Trying to count records in model %r and db_alias %r' % (model, model_db_alias)
count = queryset.count()
except DatabaseError as e:
if verbosity:
print_exc()
print "DatabaseError: Unable to count records for model '%s' (%s) because of %s." % (model.__name__, repr(model), e)
connection.close()
except:
print_exc()
print 'Connection doesnt exist?'
meta[model_name] = OrderedDict()
meta[model_name]['Meta'] = OrderedDict()
meta[model_name]['Meta']['primary_key'] = None
meta[model_name]['Meta']['count'] = count
meta[model_name]['Meta']['db_table'] = model._meta.db_table
if verbosity > 1:
print '%s.Meta = %r' % (model_name, meta[model_name]['Meta'])
# inspectdb uses: connection.introspection.get_table_description(cursor, table_name)
properties_of_fields = sql.get_meta_dicts(cursor=model_db_alias, table=meta[model_name]['Meta']['db_table'], verbosity=verbosity)
model_meta = OrderedDict((field['name'], field) for field in properties_of_fields)
if verbosity > 1:
print '-' * 20 + model_name + '-' * 20
db_primary_keys = [field['name'] for field in properties_of_fields if field['primary_key']]
if len(db_primary_keys) == 1:
meta[model_name]['Meta']['primary_key'] = db_primary_keys[0]
# augment model_meta with additional stats, but only if there are enough rows to get statistics
model_meta = augment_model_meta(model, model_db_alias, model_meta, column_name_filter=column, count=count, verbosity=verbosity)
if verbosity > 1:
print model_meta
meta[model_name].update(model_meta)
return meta
def get_db_meta(app=DEFAULT_APP_NAME,
db_alias=None,
table=None,
verbosity=0,
column=None):
"""Return a dict of dicts containing metadata about the database tables associated with an app
TODO: allow multiple apps
>>> get_db_meta('crawler', db_alias='default', table='crawler_wikiitem') # doctest: +ELLIPSIS
OrderedDict([('WikiItem', OrderedDict([('Meta', OrderedDict([('primary_key', None), ('count', 1332), ('db_table', u'crawler_wikiitem')])), (u'id', OrderedDict([('name', u'id'), ('type', ...
"""
if verbosity:
print 'Looking for app %r.' % (app, )
if app and isinstance(app, basestring):
app = djdb.get_app(app, verbosity=verbosity)
else:
app = djdb.get_app('')
model_names = list(mc.__name__ for mc in models.get_models(app))
if verbosity:
print 'Found %d models for app %r.' % (len(model_names), app)
meta = OrderedDict()
# inspectdb uses: for table_name in connection.introspection.table_names(cursor):
for model_name in model_names:
model = djdb.get_model(model_name, app=app)
if db_alias:
model_db_alias = db_alias
else:
model_db_alias = router.db_for_read(model)
queryset = model.objects
if model_db_alias:
queryset = queryset.using(model_db_alias)
if model and table is not None and isinstance(table, basestring):
if model._meta.db_table != table:
if verbosity > 1:
print 'Skipped model named %s with db table names %s.' % (
model_name, model._meta.db_table)
continue
elif callable(table):
if not table(model._meta.db_table):
if verbosity > 1:
print 'Skipped model named %s with db table names %s.' % (
model_name, model._meta.db_table)
continue
count = None
try:
if verbosity > 1:
print 'Trying to count records in model %r and db_alias %r' % (
model, model_db_alias)
count = queryset.count()
except DatabaseError as e:
if verbosity:
print_exc()
print "DatabaseError: Unable to count records for model '%s' (%s) because of %s." % (
model.__name__, repr(model), e)
connection.close()
except:
print_exc()
print 'Connection doesnt exist?'
meta[model_name] = OrderedDict()
meta[model_name]['Meta'] = OrderedDict()
meta[model_name]['Meta']['primary_key'] = None
meta[model_name]['Meta']['count'] = count
meta[model_name]['Meta']['db_table'] = model._meta.db_table
if verbosity > 1:
print '%s.Meta = %r' % (model_name, meta[model_name]['Meta'])
# inspectdb uses: connection.introspection.get_table_description(cursor, table_name)
properties_of_fields = sql.get_meta_dicts(
cursor=model_db_alias,
table=meta[model_name]['Meta']['db_table'],
verbosity=verbosity)
model_meta = OrderedDict(
(field['name'], field) for field in properties_of_fields)
if verbosity > 1:
print '-' * 20 + model_name + '-' * 20
db_primary_keys = [
field['name'] for field in properties_of_fields
if field['primary_key']
]
if len(db_primary_keys) == 1:
meta[model_name]['Meta']['primary_key'] = db_primary_keys[0]
# augment model_meta with additional stats, but only if there are enough rows to get statistics
model_meta = augment_model_meta(model,
model_db_alias,
model_meta,
column_name_filter=column,
count=count,
verbosity=verbosity)
if verbosity > 1:
print model_meta
meta[model_name].update(model_meta)
return meta
