def get_apps_from_cache(): try: from django.apps import apps return [app.models_module for app in apps.get_app_configs() if app.models_module] except ImportError: from django.db.models.loading import cache return cache.get_apps()
def handle_noargs(self, **options): if settings.MEDIA_ROOT == '': print "MEDIA_ROOT is not set, nothing to do" return # Get a list of all files under MEDIA_ROOT media = [] for root, dirs, files in os.walk(settings.MEDIA_ROOT): for f in files: media.append(os.path.abspath(os.path.join(root, f))) # Get list of all fields (value) for each model (key) # that is a FileField or subclass of a FileField model_dict = defaultdict(list) for app in cache.get_apps(): model_list = cache.get_models(app) for model in model_list: for field in model._meta.fields: if issubclass(field.__class__, models.FileField): model_dict[model].append(field) # Get a list of all files referenced in the database referenced = [] for model in model_dict.iterkeys(): all = model.objects.all().iterator() for object in all: for field in model_dict[model]: referenced.append( os.path.abspath(getattr(object, field.name).path)) # Print each file in MEDIA_ROOT that is not referenced in the database for m in media: if m not in referenced: print m
def load_disk(self): """ Loads the migrations from all INSTALLED_APPS from disk. """ self.disk_migrations = {} self.unmigrated_apps = set() self.migrated_apps = set() for app in cache.get_apps(): # Get the migrations module directory app_label = app.__name__.split(".")[-2] module_name = self.migrations_module(app_label) was_loaded = module_name in sys.modules try: module = import_module(module_name) except ImportError as e: # I hate doing this, but I don't want to squash other import errors. # Might be better to try a directory check directly. if "No module named" in str(e) and "migrations" in str(e): self.unmigrated_apps.add(app_label) continue raise else: # PY3 will happily import empty dirs as namespaces. if not hasattr(module, '__file__'): continue # Module is not a package (e.g. migrations.py). if not hasattr(module, '__path__'): continue # Force a reload if it's already loaded (tests need this) if was_loaded: six.moves.reload_module(module) self.migrated_apps.add(app_label) directory = os.path.dirname(module.__file__) # Scan for .py[c|o] files migration_names = set() for name in os.listdir(directory): if name.endswith(".py") or name.endswith(".pyc") or name.endswith(".pyo"): import_name = name.rsplit(".", 1)[0] if import_name[0] not in "_.~": migration_names.add(import_name) # Load them south_style_migrations = False for migration_name in migration_names: try: migration_module = import_module("%s.%s" % (module_name, migration_name)) except ImportError as e: # Ignore South import errors, as we're triggering them if "south" in str(e).lower(): south_style_migrations = True break raise if not hasattr(migration_module, "Migration"): raise BadMigrationError("Migration %s in app %s has no Migration class" % (migration_name, app_label)) # Ignore South-style migrations if hasattr(migration_module.Migration, "forwards"): south_style_migrations = True break self.disk_migrations[app_label, migration_name] = migration_module.Migration(migration_name, app_label) if south_style_migrations: self.unmigrated_apps.add(app_label)
def handle_noargs(self, **options): if settings.MEDIA_ROOT == '': print "MEDIA_ROOT is not set, nothing to do" return # Get a list of all files under MEDIA_ROOT media = [] for root, dirs, files in os.walk(settings.MEDIA_ROOT): for f in files: media.append(os.path.abspath(os.path.join(root, f))) # Get list of all fields (value) for each model (key) # that is a FileField or subclass of a FileField model_dict = defaultdict(list) for app in cache.get_apps(): model_list = cache.get_models(app) for model in model_list: for field in model._meta.fields: if issubclass(field.__class__, models.FileField): model_dict[model].append(field) # Get a list of all files referenced in the database referenced = [] for model in model_dict.iterkeys(): all = model.objects.all().iterator() for object in all: for field in model_dict[model]: referenced.append(os.path.abspath(getattr(object, field.name).path)) # Print each file in MEDIA_ROOT that is not referenced in the database for m in media: if m not in referenced: print m
def load_disk(self): """ Loads the migrations from all INSTALLED_APPS from disk. """ self.disk_migrations = {} self.unmigrated_apps = set() self.migrated_apps = set() for app in cache.get_apps(): # Get the migrations module directory app_label = app.__name__.split(".")[-2] module_name = self.migrations_module(app_label) try: module = import_module(module_name) except ImportError as e: # I hate doing this, but I don't want to squash other import errors. # Might be better to try a directory check directly. if "No module named" in str(e) and "migrations" in str(e): self.unmigrated_apps.add(app_label) continue self.migrated_apps.add(app_label) directory = os.path.dirname(module.__file__) # Scan for .py[c|o] files migration_names = set() for name in os.listdir(directory): if name.endswith(".py") or name.endswith(".pyc") or name.endswith(".pyo"): import_name = name.rsplit(".", 1)[0] if import_name[0] not in "_.~": migration_names.add(import_name) # Load them for migration_name in migration_names: migration_module = import_module("%s.%s" % (module_name, migration_name)) if not hasattr(migration_module, "Migration"): raise BadMigrationError("Migration %s in app %s has no Migration class" % (migration_name, app_label)) self.disk_migrations[app_label, migration_name] = migration_module.Migration(migration_name, app_label)
def get_apps(self): """ Do not run Django own tests """ return filter( lambda app: app.__name__.split('.', 1)[0] != 'django', cache.get_apps())
def find_models_with_filefield(): for app in cache.get_apps(): model_list = cache.get_models(app) for model in model_list: for field in model._meta.fields: if isinstance(field, models.FileField): pre_save.connect(remove_old_files, sender=model) post_delete.connect(remove_files, sender=model) break
def StopServices(svc_list): """ Stop a set of services. Returns the list of those that were stopped. """ old_path = [] old_environ = None if not "DJANGO_SETTINGS_MODULE" in os.environ: old_environ = True os.environ["DJANGO_SETTINGS_MODULE"] = "freenasUI.settings" if not "/usr/local/www" in sys.path: old_path.append("/usr/local/www") sys.path.append("/usr/local/www") if not "/usr/local/www/freenasUI" in sys.path: old_path.append("/usr/local/www/freenasUI") sys.path.append("/usr/local/www/freenasUI") from django.db.models.loading import cache cache.get_apps() from freenasUI.middleware.notifier import notifier n = notifier() retval = [] # Hm, this doesn't handle any particular ordering. # May need to fix this. for svc in svc_list: if not svc in SERVICES: raise ValueError("%s is not a known service" % svc) s = SERVICES[svc] svc_name = s["ServiceName"] log.debug("StopServices: svc %s maps to %s" % (svc, svc_name)) if (not s["CheckStatus"]) or n.started(svc_name): retval.append(svc) n.stop(svc_name) else: log.debug("svc %s is not started" % svc) # Should I remove the environment settings? if old_environ: os.environ.pop("DJANGO_SETTINGS_MODULE") for p in old_path: sys.path.remove(p) return retval
def find_models_with_filefield(): result = [] for app in cache.get_apps(): model_list = cache.get_models(app) for model in model_list: for field in model._meta.fields: if isinstance(field, models.FileField): result.append(model) break return result
def _get_needed_files(self): result = [] for app in cache.get_apps(): model_list = cache.get_models(app) for model in model_list: file_fields = [field.name for field in model._meta.fields if field.get_internal_type() == 'FileField'] if len(file_fields) > 0: files = model.objects.all().values_list(*file_fields) result.extend([split_name(file)[0] for file in itertools. chain.from_iterable(files) if file]) return result
def StartServices(svc_list): """ Start a set of services. THis is the output from StopServices """ old_path = [] old_environ = None if not "DJANGO_SETTINGS_MODULE" in os.environ: old_environ = True os.environ["DJANGO_SETTINGS_MODULE"] = "freenasUI.settings" if not "/usr/local/www" in sys.path: old_path.append("/usr/local/www") sys.path.append("/usr/local/www") if not "/usr/local/www/freenasUI" in sys.path: old_path.append("/usr/local/www/freenasUI") sys.path.append("/usr/local/www/freenasUI") from django.db.models.loading import cache cache.get_apps() from freenasUI.middleware.notifier import notifier n = notifier() # Hm, this doesn't handle any particular ordering. # May need to fix this. for svc in svc_list: if not svc in SERVICES: raise ValueError("%s is not a known service" % svc) svc_name = SERVICES[svc]["ServiceName"] n.start(svc_name) # Should I remove the environment settings? if old_environ: os.environ.pop("DJANGO_SETTINGS_MODULE") for p in old_path: sys.path.remove(p) return
def stashed_object_counts(request): """A context processor which adds counts of stashed objects of models which inherit from SessionStashable to RequestContext. To make a make a count appear for a particular model, set the class attribute context_count_name to an appropriate string to name its context variable, and enable this context processor in settings.py. """ extra_context = {} for app in cache.get_apps(): for model in cache.get_models(app): if issubclass(model, SessionStashable) and model.context_count_name: extra_context[model.context_count_name] = model.num_stashed_in_session(request.session) return extra_context
def _get_app_model_modules(self): """Return the entire list of registered Django app models modules. This is an internal utility function that's used to provide compatibility with all supported versions of Django. Returns: list: The list of Django applications. """ if apps: return [ app.models_module for app in apps.get_app_configs() if app.models_module ] else: return app_cache.get_apps()
def get_apps(): """Return the list of all installed apps with models. This returns the apps from the app registry on Django >= 1.7, and from the old-style cache on Django < 1.7. Returns: list: A list of all the modules containing model classes. """ if apps: # Django >= 1.7 return [ app.models_module for app in apps.get_app_configs() if app.models_module is not None ] else: # Django < 1.7 return cache.get_apps()
def _get_needed_files(self): result = [] for app in cache.get_apps(): model_list = cache.get_models(app) for model in model_list: file_fields = [ field.name for field in model._meta.fields if field.get_internal_type() == 'FileField' ] if len(file_fields) > 0: files = model.objects.all().values_list(*file_fields) result.extend([ split_name(file)[0] for file in itertools.chain.from_iterable(files) if file ]) return result
def handle_noargs(self, **options): if settings.MEDIA_ROOT == '': print("MEDIA_ROOT is not set, nothing to do") return # Get a list of all files under MEDIA_ROOT media = [] for root, dirs, files in os.walk(settings.MEDIA_ROOT): for f in files: if ('geoserver_icons' not in root) and ('resized' not in root): media.append(os.path.abspath(os.path.join(root, f))) # Get list of all fields (value) for each model (key) # that is a FileField or subclass of a FileField model_dict = defaultdict(list) for app in cache.get_apps(): model_list = cache.get_models(app) for model in model_list: for field in model._meta.fields: if issubclass(field.__class__, models.FileField): model_dict[model].append(field) # Get a list of all files referenced in the database referenced = [] for model in model_dict: all = model.objects.all().iterator() for object in all: for field in model_dict[model]: target_file = getattr(object, field.name) if target_file: referenced.append(os.path.abspath(target_file.path)) # Print each file in MEDIA_ROOT that is not referenced in the database c = 0 for m in media: if m not in referenced: print 'Removing image %s' % m os.remove(m) c = c + 1 print 'Removed %s images, from a total of %s (referenced %s)' % ( c, len(media), len(referenced))
def find_orphaned_files(path=''): """Prints a list of all files in the path that are not referenced in the database by all apps. """ if not getattr(settings, 'MEDIA_ROOT', None): sys.stdout.write('MEDIA_ROOT is not set, nothing to do') return # Get a list of all files under MEDIA_ROOT. media = set() for root, dirs, files in os.walk(os.path.join(settings.MEDIA_ROOT, path)): for f in files: media.add(os.path.abspath(os.path.join(root, f))) # Get list of all fields (value) for each model (key) # that is a FileField or subclass of a FileField. model_dict = defaultdict(list) for app in cache.get_apps(): model_list = cache.get_models(app) for model in model_list: for field in model._meta.fields: if issubclass(field.__class__, models.FileField): model_dict[model].append(field) # Get a list of all files referenced in the database. referenced = set() for model in model_dict.iterkeys(): all = model.objects.all().iterator() for object in all: for field in model_dict[model]: f = getattr(object, field.name) if f: referenced.add(os.path.abspath(f.path)) # Print each file that is not referenced in the database. for f in sorted(media - referenced): sys.stdout.write(f) sys.stdout.write('\n')
def find_orphaned_files(path=""): """Prints a list of all files in the path that are not referenced in the database by all apps. """ if not getattr(settings, "MEDIA_ROOT", None): sys.stdout.write("MEDIA_ROOT is not set, nothing to do") return # Get a list of all files under MEDIA_ROOT. media = set() for root, dirs, files in os.walk(os.path.join(settings.MEDIA_ROOT, path)): for f in files: media.add(os.path.abspath(os.path.join(root, f))) # Get list of all fields (value) for each model (key) # that is a FileField or subclass of a FileField. model_dict = defaultdict(list) for app in cache.get_apps(): model_list = cache.get_models(app) for model in model_list: for field in model._meta.fields: if issubclass(field.__class__, models.FileField): model_dict[model].append(field) # Get a list of all files referenced in the database. referenced = set() for model in model_dict.iterkeys(): all = model.objects.all().iterator() for object in all: for field in model_dict[model]: f = getattr(object, field.name) if f: referenced.add(os.path.abspath(f.path)) # Print each file that is not referenced in the database. for f in sorted(media - referenced): sys.stdout.write(f) sys.stdout.write("\n")
import datetime import logging import os import subprocess import sys sys.path.extend([ '/usr/local/www', '/usr/local/www/freenasUI', ]) os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'freenasUI.settings') # Make sure to load all modules from django.db.models.loading import cache cache.get_apps() from freenasUI.freeadmin.apppool import appPool from freenasUI.storage.models import Replication from freenasUI.common.timesubr import isTimeBetween from freenasUI.common.pipesubr import pipeopen, system from freenasUI.common.locks import mntlock from freenasUI.common.system import send_mail # DESIGN NOTES # # A snapshot transists its state in its lifetime this way: # NEW: A newly created snapshot by autosnap # LATEST: A snapshot marked to be the latest one # -: The replication system no longer cares this. #
import tdb import tempfile import time import logging from dns import resolver sys.path.extend([ '/usr/local/www', '/usr/local/www/freenasUI' ]) os.environ["DJANGO_SETTINGS_MODULE"] = "freenasUI.settings" # Make sure to load all modules from django.db.models.loading import cache cache.get_apps() from django.db.models import Q from freenasUI.account.models import ( bsdUsers, bsdGroups, bsdGroupMembership ) from freenasUI.common.freenasldap import ( FreeNAS_ActiveDirectory, FLAGS_DBINIT ) from freenasUI.common.pipesubr import pipeopen from freenasUI.common.log import log_traceback from freenasUI.common.samba import Samba4
def all_concrete_models(): return [(app, [model for model in cache.get_models(app) if not model._meta.abstract]) for app in cache.get_apps() if cache.get_models(app)]
def _stashable_models(): for app in cache.get_apps(): for model in cache.get_models(app): if issubclass(model, SessionStashable): yield model