def get_app_meta( apps=None, app_filter=lambda x: x.startswith('sec_') or x.startswith('siica_'), app_exclude_filter=None, verbosity=0, save=True): apps = apps or djdb.get_app(apps) meta = [] for app in apps: if filter_reject(app, app_filter, app_exclude_filter): continue meta += [get_db_meta(app=app, verbosity=verbosity)] if save: try: with open('db_meta_%s.json' % app, 'w') as fpout: json.dump(make_serializable(meta[-1]), fpout, indent=4) except: print_exc() if save: try: with open('db_meta_all_apps.json', 'w') as fpout: jsonifiable_data = make_serializable(meta) json.dump(jsonifiable_data, fpout, indent=4) except: print_exc() return meta
def get_db_meta(app=DEFAULT_APP_NAME, db_alias=None, table=None, verbosity=0, column=None): """Return a dict of dicts containing metadata about the database tables associated with an app TODO: allow multiple apps >>> get_db_meta('crawler', db_alias='default', table='crawler_wikiitem') # doctest: +ELLIPSIS OrderedDict([('WikiItem', OrderedDict([('Meta', OrderedDict([('primary_key', None), ('count', 1332), ('db_table', u'crawler_wikiitem')])), ('id', OrderedDict([('name', 'id'), ('type', ... """ if app and isinstance(app, basestring): app = djdb.get_app(app) else: app = djdb.get_app('') model_names = list(mc.__name__ for mc in models.get_models(app)) meta = OrderedDict() # inspectdb uses: for table_name in connection.introspection.table_names(cursor): for model_name in model_names: model = djdb.get_model(model_name, app=app) if db_alias: model_db_alias = db_alias else: model_db_alias = router.db_for_read(model) queryset = model.objects if model_db_alias: queryset = queryset.using(model_db_alias) if model and table is not None and isinstance(table, basestring): if model._meta.db_table != table: if verbosity>1: print 'skipped model named %s with db table names %s.' % (model_name, model._meta.db_table) continue elif callable(table): if not table(model._meta.db_table): if verbosity>1: print 'skipped model named %s with db table names %s.' % (model_name, model._meta.db_table) continue count = None try: print 'trying to count for model %r and db_alias %r' % (model, model_db_alias) count = queryset.count() except DatabaseError, e: if verbosity: print_exc() print "DatabaseError: Unable to count records for model '%s' (%s) because of %s." % (model.__name__, repr(model), e) connection.close() except:
def load_app_meta(apps=None, app_filter=['sec_', 'siica_'], app_exclude_filter=None): apps = apps or djdb.get_app(apps) meta = {} for app in apps: # if filter_reject(app, app_filter, app_exclude_filter): # continue if (app_filter and not any(app.startswith(af) for af in app_filter)) and ( not app_exclude_filter or not any(app.startswith(rf) for rf in app_exclude_filter)): continue with open('db_meta_%s.json' % app, 'r') as fpin: m = json.load(fpin) for table_name, table_meta in m.iteritems(): table_name = app + '.' + table_name for field_name, field_meta in table_meta.iteritems(): meta[table_name + '.' + field_name] = field_meta return meta
def load_app_meta(apps=None, app_filter=['sec_', 'siica_'], app_exclude_filter=None): apps = apps or djdb.get_app(apps) meta = {} for app in apps: # if filter_reject(app, app_filter, app_exclude_filter): # continue if (app_filter and not any(app.startswith(af) for af in app_filter) ) and (not app_exclude_filter or not any(app.startswith(rf) for rf in app_exclude_filter)): continue with open('db_meta_%s.json' % app, 'r') as fpin: m = json.load(fpin) for table_name, table_meta in m.iteritems(): table_name = app + '.' + table_name for field_name, field_meta in table_meta.iteritems(): meta[table_name + '.' + field_name] = field_meta return meta
def get_app_meta(apps=None, app_filter=lambda x: x.startswith('sec_') or x.startswith('siica_'), app_exclude_filter=None, verbosity=0, save=True): apps = apps or djdb.get_app(apps) meta = [] for app in apps: if filter_reject(app, app_filter, app_exclude_filter): continue meta += [get_db_meta(app=app, verbosity=verbosity)] if save: try: with open('db_meta_%s.json' % app, 'w') as fpout: json.dump(make_serializable(meta[-1]), fpout, indent=4) except: print_exc() if save: try: with open('db_meta_all_apps.json', 'w') as fpout: jsonifiable_data = make_serializable(meta) json.dump(jsonifiable_data, fpout, indent=4) except: print_exc() return meta
def get_db_meta(app=DEFAULT_APP_NAME, db_alias=None, table=None, verbosity=0, column=None): """Return a dict of dicts containing metadata about the database tables associated with an app TODO: allow multiple apps >>> get_db_meta('crawler', db_alias='default', table='crawler_wikiitem') # doctest: +ELLIPSIS OrderedDict([('WikiItem', OrderedDict([('Meta', OrderedDict([('primary_key', None), ('count', 1332), ('db_table', u'crawler_wikiitem')])), (u'id', OrderedDict([('name', u'id'), ('type', ... """ if verbosity: print 'Looking for app %r.' % (app, ) if app and isinstance(app, basestring): app = djdb.get_app(app, verbosity=verbosity) else: app = djdb.get_app('') model_names = list(mc.__name__ for mc in models.get_models(app)) if verbosity: print 'Found %d models for app %r.' % (len(model_names), app) meta = OrderedDict() # inspectdb uses: for table_name in connection.introspection.table_names(cursor): for model_name in model_names: model = djdb.get_model(model_name, app=app) if db_alias: model_db_alias = db_alias else: model_db_alias = router.db_for_read(model) queryset = model.objects if model_db_alias: queryset = queryset.using(model_db_alias) if model and table is not None and isinstance(table, basestring): if model._meta.db_table != table: if verbosity>1: print 'Skipped model named %s with db table names %s.' % (model_name, model._meta.db_table) continue elif callable(table): if not table(model._meta.db_table): if verbosity>1: print 'Skipped model named %s with db table names %s.' % (model_name, model._meta.db_table) continue count = None try: if verbosity > 1: print 'Trying to count records in model %r and db_alias %r' % (model, model_db_alias) count = queryset.count() except DatabaseError as e: if verbosity: print_exc() print "DatabaseError: Unable to count records for model '%s' (%s) because of %s." % (model.__name__, repr(model), e) connection.close() except: print_exc() print 'Connection doesnt exist?' meta[model_name] = OrderedDict() meta[model_name]['Meta'] = OrderedDict() meta[model_name]['Meta']['primary_key'] = None meta[model_name]['Meta']['count'] = count meta[model_name]['Meta']['db_table'] = model._meta.db_table if verbosity > 1: print '%s.Meta = %r' % (model_name, meta[model_name]['Meta']) # inspectdb uses: connection.introspection.get_table_description(cursor, table_name) properties_of_fields = sql.get_meta_dicts(cursor=model_db_alias, table=meta[model_name]['Meta']['db_table'], verbosity=verbosity) model_meta = OrderedDict((field['name'], field) for field in properties_of_fields) if verbosity > 1: print '-' * 20 + model_name + '-' * 20 db_primary_keys = [field['name'] for field in properties_of_fields if field['primary_key']] if len(db_primary_keys) == 1: meta[model_name]['Meta']['primary_key'] = db_primary_keys[0] # augment model_meta with additional stats, but only if there are enough rows to get statistics model_meta = augment_model_meta(model, model_db_alias, model_meta, column_name_filter=column, count=count, verbosity=verbosity) if verbosity > 1: print model_meta meta[model_name].update(model_meta) return meta
def get_db_meta(app=DEFAULT_APP_NAME, db_alias=None, table=None, verbosity=0, column=None): """Return a dict of dicts containing metadata about the database tables associated with an app TODO: allow multiple apps >>> get_db_meta('crawler', db_alias='default', table='crawler_wikiitem') # doctest: +ELLIPSIS OrderedDict([('WikiItem', OrderedDict([('Meta', OrderedDict([('primary_key', None), ('count', 1332), ('db_table', u'crawler_wikiitem')])), (u'id', OrderedDict([('name', u'id'), ('type', ... """ if verbosity: print 'Looking for app %r.' % (app, ) if app and isinstance(app, basestring): app = djdb.get_app(app, verbosity=verbosity) else: app = djdb.get_app('') model_names = list(mc.__name__ for mc in models.get_models(app)) if verbosity: print 'Found %d models for app %r.' % (len(model_names), app) meta = OrderedDict() # inspectdb uses: for table_name in connection.introspection.table_names(cursor): for model_name in model_names: model = djdb.get_model(model_name, app=app) if db_alias: model_db_alias = db_alias else: model_db_alias = router.db_for_read(model) queryset = model.objects if model_db_alias: queryset = queryset.using(model_db_alias) if model and table is not None and isinstance(table, basestring): if model._meta.db_table != table: if verbosity > 1: print 'Skipped model named %s with db table names %s.' % ( model_name, model._meta.db_table) continue elif callable(table): if not table(model._meta.db_table): if verbosity > 1: print 'Skipped model named %s with db table names %s.' % ( model_name, model._meta.db_table) continue count = None try: if verbosity > 1: print 'Trying to count records in model %r and db_alias %r' % ( model, model_db_alias) count = queryset.count() except DatabaseError as e: if verbosity: print_exc() print "DatabaseError: Unable to count records for model '%s' (%s) because of %s." % ( model.__name__, repr(model), e) connection.close() except: print_exc() print 'Connection doesnt exist?' meta[model_name] = OrderedDict() meta[model_name]['Meta'] = OrderedDict() meta[model_name]['Meta']['primary_key'] = None meta[model_name]['Meta']['count'] = count meta[model_name]['Meta']['db_table'] = model._meta.db_table if verbosity > 1: print '%s.Meta = %r' % (model_name, meta[model_name]['Meta']) # inspectdb uses: connection.introspection.get_table_description(cursor, table_name) properties_of_fields = sql.get_meta_dicts( cursor=model_db_alias, table=meta[model_name]['Meta']['db_table'], verbosity=verbosity) model_meta = OrderedDict( (field['name'], field) for field in properties_of_fields) if verbosity > 1: print '-' * 20 + model_name + '-' * 20 db_primary_keys = [ field['name'] for field in properties_of_fields if field['primary_key'] ] if len(db_primary_keys) == 1: meta[model_name]['Meta']['primary_key'] = db_primary_keys[0] # augment model_meta with additional stats, but only if there are enough rows to get statistics model_meta = augment_model_meta(model, model_db_alias, model_meta, column_name_filter=column, count=count, verbosity=verbosity) if verbosity > 1: print model_meta meta[model_name].update(model_meta) return meta