def load_label(self, fixture_label, app_fixtures): parts = fixture_label.split(".") if len(parts) > 1 and parts[-1] in self.compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = self.compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = ".".join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) else: raise CommandError( "Problem installing fixture '%s': %s is not a known serialization format." % (fixture_name, format) ) if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [""] for fixture_dir in fixture_dirs: self.process_dir(fixture_dir, fixture_name, compression_formats, formats)
def deserialize_objects(self, fixture, database, verbosity): """ Adapted from 'loaddata' django command Returns a list of objects loaded from a single fixture; no comporession accepted at the moment """ objects_list = [] humanize = lambda dirname: "'%s'" % dirname if dirname else 'absolute path' parts = fixture.split('.') if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity >= 2: self.stdout.write("Checking '%s' fixture..." % fixture_name) else: raise CommandError( "Problem deserializing fixture '%s': %s is not a known serialization format." % (fixture_name, format)) fixture_dir = fixture_name format = formats[0] full_path = '.'.join([fixture_dir, format]) open_method = open try: fixture = open_method(full_path, 'r') except IOError: if verbosity >= 2: self.stdout.write("No %s fixture '%s' in %s." % (format, fixture_name, humanize(fixture_dir))) else: try: objects = serializers.deserialize(format, fixture, using=database, ignorenonexistent=False) for object in objects: objects_list.append(object) except Exception as e: if not isinstance(e, CommandError): e.args = ("Problem deserializing fixture '%s': %s" % (full_path, e),) raise finally: fixture.close() return objects_list
def handle(self, *app_labels, **options): from django.db.models import get_app, get_apps, get_models format = options.get("format", "json") indent = options.get("indent", None) show_traceback = options.get("traceback", False) if len(app_labels) == 0: app_list = get_apps() else: app_list = [get_app(app_label) for app_label in app_labels] # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) objects = [] for app in app_list: for model in get_models(app): objects.extend(model._default_manager.all()) try: return serializers.serialize(format, objects, indent=indent) except Exception, e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def loaddata(self, fixture_labels): connection = connections[self.using] # Keep a count of the installed objects and fixtures self.fixture_count = 0 self.loaded_object_count = 0 self.fixture_object_count = 0 self.models = set() self.serialization_formats = serializers.get_public_serializer_formats() # Forcing binary mode may be revisited after dropping Python 2 support (see #22399) self.compression_formats = { None: (open, 'rb'), 'gz': (gzip.GzipFile, 'rb'), 'zip': (SingleZipReader, 'r'), } if has_bz2: self.compression_formats['bz2'] = (bz2.BZ2File, 'r') # Django's test suite repeatedly tries to load initial_data fixtures # from apps that don't have any fixtures. Because disabling constraint # checks can be expensive on some database (especially MSSQL), bail # out early if no fixtures are found. for fixture_label in fixture_labels: if self.find_fixtures(fixture_label): break else: return with connection.constraint_checks_disabled(): for fixture_label in fixture_labels: self.load_label(fixture_label) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in self.models] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem installing fixtures: %s" % e,) raise # If we found even one object in a fixture, we need to reset the # database sequences. if self.loaded_object_count > 0: sequence_sql = connection.ops.sequence_reset_sql(no_style(), self.models) if sequence_sql: if self.verbosity >= 2: self.stdout.write("Resetting sequences\n") with connection.cursor() as cursor: for line in sequence_sql: cursor.execute(line) if self.verbosity >= 1: if self.fixture_object_count == self.loaded_object_count: self.stdout.write("Installed %d object(s) from %d fixture(s)" % (self.loaded_object_count, self.fixture_count)) else: self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)" % (self.loaded_object_count, self.fixture_object_count, self.fixture_count))
def handle(self, *args, **options): namespaces = options.get('namespaces', []) show_traceback = options.get('traceback', False) format = options.get('format', 'json') if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) objects = [] if not namespaces: objects.extend(models.Schema.objects.all()) objects.extend(models.ParameterName.objects.all()) else: schemas = models.Schema.objects.filter(namespace__in=namespaces) if not schemas: raise CommandError('No schemas found') schema_set = set([s.namespace for s in schemas]) arg_set = set(args) skipped = arg_set - schema_set if skipped: sys.stderr.write('Schema not found: {0}\n'.format( ', '.join(skipped))) objects.extend(schemas) objects.extend(models.ParameterName.objects.filter(schema__namespace__in=args)) try: return serializers.serialize( format, objects, indent=4, use_natural_foreign_keys=True) except Exception as e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def test_unregister(self): "Unregistering a serializer doesn't cause the registry to be repopulated. Refs #14823" serializers.unregister_serializer('xliff') public_formats = serializers.get_public_serializer_formats() self.assertNotIn('xliff', public_formats)
def find_fixtures(self, fixture_label='aristotle_help_files'): """ Finds fixture files for a given label. """ fixture_name, ser_fmt = self.parse_name(fixture_label) databases = [self.using, None] ser_fmts = serializers.get_public_serializer_formats() if ser_fmt is None else [ser_fmt] if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) if os.path.isabs(fixture_name): fixture_dirs = [os.path.dirname(fixture_name)] else: fixture_dirs = self.fixture_dirs if os.path.sep in os.path.normpath(fixture_name): fixture_dirs = [os.path.join(dir_, os.path.dirname(fixture_name)) for dir_ in fixture_dirs] suffixes = ( '.'.join(ext for ext in combo if ext) for combo in product(databases, ser_fmts) ) fixture_name = "*" search_name = "" targets = set('.'.join((search_name, suffix)) for suffix in suffixes) fixture_files = [] for fixture_dir in fixture_dirs: if self.verbosity >= 2: self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) fixture_files_in_dir = [] for dir_name, sub, candidates in os.walk(fixture_dir): # , search_name + '*')): for candidate in candidates: candidate = os.path.join(dir_name, candidate) if any([os.path.basename(candidate).endswith(t) for t in targets]): # Save the fixture_dir and fixture_name for future error messages. fixture_files_in_dir.append((candidate, fixture_dir, candidate.split('/')[-1])) dest_static_dir = os.path.join(settings.STATIC_ROOT, "aristotle_help") src = os.path.join(fixture_dir, 'static') if not os.path.exists(dest_static_dir): os.makedirs(dest_static_dir) if os.path.exists(src): from distutils import dir_util dir_util.copy_tree(src, dest_static_dir) # Check kept for backwards-compatibility; it isn't clear why # duplicates are only allowed in different directories. # Commented out from django # if len(fixture_files_in_dir) > 1: # raise CommandError( # "Multiple fixtures named '%s' in %s. Aborting." % # (fixture_name, humanize(fixture_dir))) fixture_files.extend(fixture_files_in_dir) return fixture_files
def handle_noargs(self, **options): """ Generates a dump of Django CMS database. """ format = options.get('format') indent = options.get('indent') use_natural_keys = options.get('use_natural_keys') show_traceback = options.get('traceback') # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise base.CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise base.CommandError("Unknown serialization format: %s" % format) pages = cms_models.Page.objects.filter(published=True) placeholders = cms_models.Placeholder.objects.filter(page__in=pages) blog_entries = blog_models.Entry.published.all() blog_placeholders = cms_models.Placeholder.objects.filter(entry__in=blog_entries) plugins = cms_models.CMSPlugin.objects.filter(Q(placeholder__in=placeholders) | Q(placeholder__in=blog_placeholders)) def plugin_objects(): for plugin in plugins: instance = plugin.get_plugin_instance()[0] if instance is None: raise base.CommandError("Invalid plugin instance: %s from placeholder %s" % (plugin.pk, plugin.placeholder.pk)) yield instance objects = itertools.chain( sites_models.Site.objects.all(), contenttypes_models.ContentType.objects.all(), placeholders, pages, cms_models.Title.objects.filter(page__in=pages), blog_placeholders, blog_entries, blog_models.EntryTitle.objects.filter(entry__in=blog_entries), plugins, plugin_objects(), filer_models.Folder.objects.all(), filer_models.File.objects.non_polymorphic().filter(is_public=True), filer_models.Image.objects.non_polymorphic().filter(is_public=True), filer_image_models.ThumbnailOption.objects.all(), snippet_models.Snippet.objects.all(), ) try: return serializers.serialize(format, objects, indent=indent, use_natural_keys=use_natural_keys) except Exception, e: if show_traceback: raise raise base.CommandError("Unable to serialize database: %s" % e)
def handle_models(self, models, **options): format = options.get('format','json') indent = options.get('indent',None) show_traceback = options.get('traceback', False) propagate = options.get('propagate', True) # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) objects = [] for model, slice in models: if isinstance(slice, basestring): objects.extend(model._default_manager.filter(pk__exact=slice)) elif not slice or type(slice) is list: items = model._default_manager.all() if slice and slice[0]: items = items.filter(pk__gte=slice[0]) if slice and slice[1]: items = items.filter(pk__lt=slice[1]) items = items.order_by(model._meta.pk.attname) objects.extend(items) else: raise CommandError("Wrong slice: %s" % slice) all = objects if propagate: collected = set([(x.__class__, x.pk) for x in all]) while objects: related = [] for x in objects: if DEBUG: print "Adding %s[%s]" % (model_name(x), x.pk) for f in x.__class__._meta.fields + x.__class__._meta.many_to_many: if isinstance(f, ForeignKey): new = getattr(x, f.name) # instantiate object if new and not (new.__class__, new.pk) in collected: collected.add((new.__class__, new.pk)) related.append(new) if isinstance(f, ManyToManyField): for new in getattr(x, f.name).all(): if new and not (new.__class__, new.pk) in collected: collected.add((new.__class__, new.pk)) related.append(new) objects = related all.extend(objects) try: return serializers.serialize(format, all, indent=indent) except Exception, e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def test_register(self): "Registering a new serializer populates the full registry. Refs #14823" serializers.register_serializer("json3", "django.core.serializers.json") public_formats = serializers.get_public_serializer_formats() self.assertIn("json3", public_formats) self.assertIn("json2", public_formats) self.assertIn("xml", public_formats)
def test_register(self): "Registering a new serializer populates the full registry. Refs #14823" serializers.register_serializer('json3', 'django.core.serializers.json') public_formats = serializers.get_public_serializer_formats() self.assertIn('json3', public_formats) self.assertIn('json2', public_formats) self.assertIn('xml', public_formats)
def find_fixtures(self, fixture_label): """ Finds fixture files for a given label. """ fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label) databases = [self.using, None] cmp_fmts = list(self.compression_formats.keys()) if cmp_fmt is None else [cmp_fmt] ser_fmts = serializers.get_public_serializer_formats() if ser_fmt is None else [ser_fmt] if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) if os.path.isabs(fixture_name): fixture_dirs = [os.path.dirname(fixture_name)] fixture_name = os.path.basename(fixture_name) else: fixture_dirs = self.fixture_dirs if os.path.sep in fixture_name: fixture_dirs = [os.path.join(dir_, os.path.dirname(fixture_name)) for dir_ in fixture_dirs] fixture_name = os.path.basename(fixture_name) suffixes = ('.'.join(ext for ext in combo if ext) for combo in product(databases, ser_fmts, cmp_fmts)) targets = set('.'.join((fixture_name, suffix)) for suffix in suffixes) fixture_files = [] for fixture_dir in fixture_dirs: if self.verbosity >= 2: self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) fixture_files_in_dir = [] for candidate in glob.iglob(os.path.join(fixture_dir, fixture_name + '*')): if os.path.basename(candidate) in targets: # Save the fixture_dir and fixture_name for future error messages. fixture_files_in_dir.append((candidate, fixture_dir, fixture_name)) if self.verbosity >= 2 and not fixture_files_in_dir: self.stdout.write("No fixture '%s' in %s." % (fixture_name, humanize(fixture_dir))) # Check kept for backwards-compatibility; it isn't clear why # duplicates are only allowed in different directories. if len(fixture_files_in_dir) > 1: raise CommandError( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) fixture_files.extend(fixture_files_in_dir) if fixture_name != 'initial_data' and not fixture_files: # Warning kept for backwards-compatibility; why not an exception? warnings.warn("No fixture named '%s' found." % fixture_name) elif fixture_name == 'initial_data' and fixture_files: warnings.warn( 'initial_data fixtures are deprecated. Use data migrations instead.', RemovedInDjango19Warning ) return fixture_files
def find_fixtures(self, fixture_label): """ Finds fixture files for a given label. """ fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label) databases = [self.using, None] cmp_fmts = list(self.compression_formats.keys()) if cmp_fmt is None else [cmp_fmt] ser_fmts = serializers.get_public_serializer_formats() if ser_fmt is None else [ser_fmt] if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) if os.path.isabs(fixture_name): fixture_dirs = [os.path.dirname(fixture_name)] fixture_name = os.path.basename(fixture_name) else: fixture_dirs = self.fixture_dirs if os.path.sep in os.path.normpath(fixture_name): fixture_dirs = [os.path.join(dir_, os.path.dirname(fixture_name)) for dir_ in fixture_dirs] fixture_name = os.path.basename(fixture_name) suffixes = ( '.'.join(ext for ext in combo if ext) for combo in product(databases, ser_fmts, cmp_fmts) ) if fixture_name == "*": search_name = "" else: search_name = fixture_name targets = set('.'.join((search_name, suffix)) for suffix in suffixes) fixture_files = [] for fixture_dir in fixture_dirs: if self.verbosity >= 2: self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) fixture_files_in_dir = [] for candidate in glob.iglob(os.path.join(fixture_dir, search_name + '*')): if any([os.path.basename(candidate).endswith(t) for t in targets]): # Save the fixture_dir and fixture_name for future error messages. fixture_files_in_dir.append((candidate, fixture_dir, fixture_name)) if self.verbosity >= 2 and not fixture_files_in_dir: self.stdout.write("No fixture '%s' in %s." % (fixture_name, humanize(fixture_dir))) # Check kept for backwards-compatibility; it isn't clear why # duplicates are only allowed in different directories. # Commented out from django # if len(fixture_files_in_dir) > 1: # raise CommandError( # "Multiple fixtures named '%s' in %s. Aborting." % # (fixture_name, humanize(fixture_dir))) fixture_files.extend(fixture_files_in_dir) return fixture_files
def find_fixtures(self, fixture_label): """Find fixture files for a given label.""" fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label) databases = [self.using, None] cmp_fmts = list( self.compression_formats.keys()) if cmp_fmt is None else [cmp_fmt] ser_fmts = serializers.get_public_serializer_formats( ) if ser_fmt is None else [ser_fmt] if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) if os.path.isabs(fixture_name): fixture_dirs = [os.path.dirname(fixture_name)] fixture_name = os.path.basename(fixture_name) else: fixture_dirs = self.fixture_dirs if os.path.sep in os.path.normpath(fixture_name): fixture_dirs = [ os.path.join(dir_, os.path.dirname(fixture_name)) for dir_ in fixture_dirs ] fixture_name = os.path.basename(fixture_name) suffixes = ('.'.join(ext for ext in combo if ext) for combo in product(databases, ser_fmts, cmp_fmts)) targets = set('.'.join((fixture_name, suffix)) for suffix in suffixes) fixture_files = [] for fixture_dir in fixture_dirs: if self.verbosity >= 2: self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) fixture_files_in_dir = [] path = os.path.join(fixture_dir, fixture_name) for candidate in glob.iglob(glob.escape(path) + '*'): if os.path.basename(candidate) in targets: # Save the fixture_dir and fixture_name for future error messages. fixture_files_in_dir.append( (candidate, fixture_dir, fixture_name)) if self.verbosity >= 2 and not fixture_files_in_dir: self.stdout.write("No fixture '%s' in %s." % (fixture_name, humanize(fixture_dir))) # Check kept for backwards-compatibility; it isn't clear why # duplicates are only allowed in different directories. if len(fixture_files_in_dir) > 1: raise CommandError( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) fixture_files.extend(fixture_files_in_dir) if not fixture_files: raise CommandError("No fixture named '%s' found." % fixture_name) return fixture_files
def test_unregister(self): "Unregistering a serializer doesn't cause the registry to be repopulated. Refs #14823" serializers.unregister_serializer('xml') serializers.register_serializer('json3', 'django.core.serializers.json') public_formats = serializers.get_public_serializer_formats() self.assertNotIn('xml', public_formats) self.assertIn('json3', public_formats)
def test_register(self): "Registering a new serializer populates the full registry. Refs #14823" serializers.register_serializer('json3', 'django.core.serializers.json') public_formats = serializers.get_public_serializer_formats() self.assertIn('json3', public_formats) self.assertIn('json2', public_formats) self.assertIn('xml', public_formats)
def loaddata(self, fixture_labels): connection = connections[self.using] # Keep a count of the installed objects and fixtures self.fixture_count = 0 self.loaded_object_count = 0 self.fixture_object_count = 0 self.models = set() self.serialization_formats = serializers.get_public_serializer_formats( ) # Forcing binary mode may be revisited after dropping Python 2 support (see #22399) self.compression_formats = { None: (open, 'rb'), 'gz': (gzip.GzipFile, 'rb'), 'zip': (SingleZipReader, 'r'), } if has_bz2: self.compression_formats['bz2'] = (bz2.BZ2File, 'r') with connection.constraint_checks_disabled(): for fixture_label in fixture_labels: self.load_label(fixture_label) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in self.models] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem installing fixtures: %s" % e, ) raise # If we found even one object in a fixture, we need to reset the # database sequences. if self.loaded_object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( no_style(), self.models) if sequence_sql: if self.verbosity >= 2: self.stdout.write("Resetting sequences\n") with connection.cursor() as cursor: for line in sequence_sql: cursor.execute(line) if self.verbosity >= 1: if self.fixture_count == 0 and self.hide_empty: pass elif self.fixture_object_count == self.loaded_object_count: self.stdout.write( "Installed %d object(s) from %d fixture(s)" % (self.loaded_object_count, self.fixture_count)) else: self.stdout.write( "Installed %d object(s) (of %d) from %d fixture(s)" % (self.loaded_object_count, self.fixture_object_count, self.fixture_count))
def test_builtin_serializers(self): """ 'geojson' should be listed in available serializers. """ all_formats = set(serializers.get_serializer_formats()) public_formats = set(serializers.get_public_serializer_formats()) self.assertIn('geojson', all_formats), self.assertIn('geojson', public_formats)
def test_builtin_serializers(self): """ 'geojson' should be listed in available serializers. """ all_formats = set(serializers.get_serializer_formats()) public_formats = set(serializers.get_public_serializer_formats()) self.assertIn('geojson', all_formats), self.assertIn('geojson', public_formats)
def test_unregister(self): "Unregistering a serializer doesn't cause the registry to be repopulated. Refs #14823" serializers.unregister_serializer('xml') serializers.register_serializer('json3', 'django.core.serializers.json') public_formats = serializers.get_public_serializer_formats() self.assertNotIn('xml', public_formats) self.assertIn('json3', public_formats)
def _find_fixtures(self, fixture_label): """ Finds fixture files for a given label. """ fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label) databases = [self.using, None] cmp_fmts = list(self.compression_formats.keys()) \ if cmp_fmt is None else [cmp_fmt] ser_fmts = serializers.get_public_serializer_formats() \ if ser_fmt is None else [ser_fmt] if os.path.isabs(fixture_name): fixture_dirs = [os.path.dirname(fixture_name)] fixture_name = os.path.basename(fixture_name) else: fixture_dirs = self.fixture_dirs if os.path.sep in os.path.normpath(fixture_name): fixture_dirs = [ os.path.join(dir_, os.path.dirname(fixture_name)) for dir_ in fixture_dirs] fixture_name = os.path.basename(fixture_name) suffixes = ('.'.join(ext for ext in combo if ext) for combo in product(databases, ser_fmts, cmp_fmts)) targets = set('.'.join((fixture_name, suffix)) for suffix in suffixes) fixture_files = [] for fixture_dir in fixture_dirs: logger.debug(" Checking %s for fixtures..." % humanize(fixture_dir)) fixture_files_in_dir = [] path = os.path.join(fixture_dir, fixture_name) for candidate in glob.iglob(glob_escape(path) + '*'): if os.path.basename(candidate) in targets: # Save the fixture_dir and fixture_name for future # error messages. logger.debug(" Found fixture: %s" % candidate) fixture_files_in_dir.append((candidate, fixture_dir, fixture_name)) if not fixture_files_in_dir: logger.debug(" No fixture found") # Check kept for backwards-compatibility; it isn't clear why # duplicates are only allowed in different directories. if len(fixture_files_in_dir) > 1: raise CommandError( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) fixture_files.extend(fixture_files_in_dir) if not fixture_files: # Warning kept for backwards-compatibility; why not an exception? warnings.warn("No fixture named '%s' found." % fixture_name) return fixture_files
def find_fixtures(self, fixture_label): """Find fixture files for a given label.""" if fixture_label == READ_STDIN: return [(READ_STDIN, None, READ_STDIN)] fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label) databases = [self.using, None] cmp_fmts = list(self.compression_formats) if cmp_fmt is None else [cmp_fmt] ser_fmts = serializers.get_public_serializer_formats() if ser_fmt is None else [ser_fmt] if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) if os.path.isabs(fixture_name): fixture_dirs = [os.path.dirname(fixture_name)] fixture_name = os.path.basename(fixture_name) else: fixture_dirs = self.fixture_dirs if os.path.sep in os.path.normpath(fixture_name): fixture_dirs = [os.path.join(dir_, os.path.dirname(fixture_name)) for dir_ in fixture_dirs] fixture_name = os.path.basename(fixture_name) suffixes = ( '.'.join(ext for ext in combo if ext) for combo in product(databases, ser_fmts, cmp_fmts) ) targets = {'.'.join((fixture_name, suffix)) for suffix in suffixes} fixture_files = [] for fixture_dir in fixture_dirs: if self.verbosity >= 2: self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) fixture_files_in_dir = [] path = os.path.join(fixture_dir, fixture_name) for candidate in glob.iglob(glob.escape(path) + '*'): if os.path.basename(candidate) in targets: # Save the fixture_dir and fixture_name for future error messages. fixture_files_in_dir.append((candidate, fixture_dir, fixture_name)) if self.verbosity >= 2 and not fixture_files_in_dir: self.stdout.write("No fixture '%s' in %s." % (fixture_name, humanize(fixture_dir))) # Check kept for backwards-compatibility; it isn't clear why # duplicates are only allowed in different directories. if len(fixture_files_in_dir) > 1: raise CommandError( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) fixture_files.extend(fixture_files_in_dir) if not fixture_files: raise CommandError("No fixture named '%s' found." % fixture_name) return fixture_files
def _find_fixtures(self, fixture_label): """ Finds fixture files for a given label. """ fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label) databases = [self.using, None] cmp_fmts = list(self.compression_formats.keys()) if cmp_fmt is None else [cmp_fmt] ser_fmts = serializers.get_public_serializer_formats() if ser_fmt is None else [ser_fmt] # Check kept for backwards-compatibility; it doesn't look very useful. if '.' in os.path.basename(fixture_name): raise CommandError( "Problem installing fixture '%s': %s is not a known " "serialization format." % tuple(fixture_name.rsplit('.'))) if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) if os.path.isabs(fixture_name): fixture_dirs = [os.path.dirname(fixture_name)] fixture_name = os.path.basename(fixture_name) else: fixture_dirs = self.fixture_dirs suffixes = ('.'.join(ext for ext in combo if ext) for combo in product(databases, ser_fmts, cmp_fmts)) targets = set('.'.join((fixture_name, suffix)) for suffix in suffixes) fixture_files = [] for fixture_dir in fixture_dirs: if self.verbosity >= 2: self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) fixture_files_in_dir = [] for candidate in glob.iglob(os.path.join(fixture_dir, fixture_name + '*')): if os.path.basename(candidate) in targets: # Save the fixture_dir and fixture_name for future error messages. fixture_files_in_dir.append((candidate, fixture_dir, fixture_name)) if self.verbosity >= 2 and not fixture_files_in_dir: self.stdout.write("No fixture '%s' in %s." % (fixture_name, humanize(fixture_dir))) # Check kept for backwards-compatibility; it isn't clear why # duplicates are only allowed in different directories. if len(fixture_files_in_dir) > 1: raise CommandError( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) fixture_files.extend(fixture_files_in_dir) if fixture_name != 'initial_data' and not fixture_files: # Warning kept for backwards-compatibility; why not an exception? warnings.warn("No fixture named '%s' found." % fixture_name) return fixture_files
def handle(self, *app_labels, **options): format = options.get('format','json') indent = options.get('indent',None) exclude = options.get('exclude',[]) exclude_models = options.get('excludem',[]) #includem = options.get('includem',[]) show_traceback = options.get('extrasort', False) extrasort = options.get('traceback', False) excluded_apps = [get_app(app_label) for app_label in exclude] excluded_models = [(get_app(app_label.split('.')[0]), get_model(app_label.split('.')[0], app_label.split('.')[1]) ) for app_label in exclude_models] if len(app_labels) == 0: app_list = [app for app in get_apps() if app not in excluded_apps] else: app_list = [get_app(app_label) for app_label in app_labels] # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) objects = [] models=[] for app in app_list: for model in get_models(app): if intuplelist0(excluded_models, app) and\ intuplelist1(excluded_models, model): pass else: models.append(model) if extrasort: models = sort_by_relation(models) for model in models: try: objects.extend(model._default_manager.all().order_by(pk)) except: objects.extend(model._default_manager.all()) try: return serializers.serialize(format, objects, indent=indent) except Exception, e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def loadhelp(self, fixture_labels=['aristotle_help_files']): connection = connections[self.using] # Keep a count of the installed objects and fixtures self.fixture_count = 0 self.loaded_object_count = 0 self.fixture_object_count = 0 self.models = set() self.serialization_formats = serializers.get_public_serializer_formats( ) # Django's test suite repeatedly tries to load initial_data fixtures # from apps that don't have any fixtures. Because disabling constraint # checks can be expensive on some database (especially MSSQL), bail # out early if no fixtures are found. if not self.find_fixtures(): return with connection.constraint_checks_disabled(): self.load_label() # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in self.models] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem installing fixtures: %s" % e, ) raise # If we found even one object in a fixture, we need to reset the # database sequences. if self.loaded_object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( no_style(), self.models) if sequence_sql: if self.verbosity >= 2: self.stdout.write("Resetting sequences\n") with connection.cursor() as cursor: for line in sequence_sql: cursor.execute(line) if self.verbosity >= 1: if self.fixture_count == 0 and self.hide_empty: pass elif self.fixture_object_count == self.loaded_object_count: self.stdout.write( "Installed %d object(s) from %d fixture(s)" % (self.loaded_object_count, self.fixture_count)) else: self.stdout.write( "Installed %d object(s) (of %d) from %d fixture(s)" % (self.loaded_object_count, self.fixture_object_count, self.fixture_count))
def _find_fixtures(self, fixture_label): """ Finds fixture files for a given label. """ fixture_name, ser_fmt, cmp_fmt = self.parse_name(fixture_label) databases = [self.using, None] cmp_fmts = list(self.compression_formats.keys()) if cmp_fmt is None else [cmp_fmt] ser_fmts = serializers.get_public_serializer_formats() if ser_fmt is None else [ser_fmt] # Check kept for backwards-compatibility; it doesn't look very useful. if "." in os.path.basename(fixture_name): raise CommandError( "Problem installing fixture '%s': %s is not a known " "serialization format." % tuple(fixture_name.rsplit(".")) ) if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) if os.path.sep in fixture_name: fixture_dirs = [os.path.dirname(fixture_name)] fixture_name = os.path.basename(fixture_name) else: fixture_dirs = self.fixture_dirs suffixes = (".".join(ext for ext in combo if ext) for combo in product(databases, ser_fmts, cmp_fmts)) targets = set(".".join((fixture_name, suffix)) for suffix in suffixes) fixture_files = [] for fixture_dir in fixture_dirs: if self.verbosity >= 2: self.stdout.write("Checking %s for fixtures..." % humanize(fixture_dir)) fixture_files_in_dir = [] for candidate in glob.iglob(os.path.join(fixture_dir, fixture_name + "*")): if os.path.basename(candidate) in targets: # Save the fixture_dir and fixture_name for future error messages. fixture_files_in_dir.append((candidate, fixture_dir, fixture_name)) if self.verbosity >= 2 and not fixture_files_in_dir: self.stdout.write("No fixture '%s' in %s." % (fixture_name, humanize(fixture_dir))) # Check kept for backwards-compatibility; it isn't clear why # duplicates are only allowed in different directories. if len(fixture_files_in_dir) > 1: raise CommandError( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir)) ) fixture_files.extend(fixture_files_in_dir) if fixture_name != "initial_data" and not fixture_files: # Warning kept for backwards-compatibility; why not an exception? warnings.warn("No fixture named '%s' found." % fixture_name) return fixture_files
def load_label(self, fixture_label, app_fixtures): parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in self.compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = self.compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if self.verbosity >= 2: self.stdout.write("Loading '%s' fixtures..." % fixture_name) else: raise CommandError( "Problem installing fixture '%s': %s is not a known serialization format." % (fixture_name, format)) if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [''] label_found = False for fixture_dir in fixture_dirs: found = self.process_dir(fixture_dir, fixture_name, compression_formats, formats) label_found = label_found or found if fixture_name != 'initial_data' and not label_found: warnings.warn("No fixture named '%s' found." % fixture_name)
def load_fixture(self, fixture): parts = fixture.split(".") if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = ".".join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if not formats: raise Exception( "Problem installing fixture '%s': %s is not a known serialization format.\n" % (fixture_name, format) ) full_path = self.get_fixture_path(fixture) try: data = open(full_path, "r") return serializers.deserialize(format, data) finally: fixture.close()
def handle(self, *args, **options): if len(args) != 1: raise CommandError('You must use specify a unique object app_label.Model.field:value properly') primary_object_str = args[0] if not primary_object_regex(primary_object_str): raise CommandError('You must use specify app_label.Model.field:value properly') exclude_models_args = options.get('exclude', []) include_models_args = options.get('include', []) if exclude_models_args and include_models_args: raise CommandError('You cannot use --exclude and --include options at the same time') exclude = exclude_models_args.split(',') if exclude_models_args else [] include = include_models_args.split(',') if include_models_args else [] output_format = options.get('output_format') if output_format not in serializers.get_public_serializer_formats(): raise CommandError('Unknown serialization format: {}'.format(output_format)) output_mode = options.get('output_mode') database = options.get('database') indent = options.get('indent') primary_object = get_primary_object(primary_object_str) if include: models_to_include = [get_app_model(app_model) for app_model in include] data_to_serialize = get_data(primary_object, database, 'include', models_to_include) else: models_to_exclude = [get_app_model(app_model) for app_model in exclude] data_to_serialize = get_data(primary_object, database, 'exclude', models_to_exclude) if output_mode not in ('directory', 'file'): raise CommandError('You must specify an output mode to dump the fixtures') valid_path = self.create_directories() if output_mode == 'directory': organized_data = data_by_model(data_to_serialize) for model, data in organized_data.iteritems(): filename = '{filename}.{extension}'.format(filename=model._meta.model_name, extension=output_format) with open(os.path.join(valid_path, filename), 'w') as stream: serializers.serialize(output_format, data, indent=indent, stream=stream) elif output_mode == 'file': filename = 'objectdump.{extension}'.format(extension=output_format) with open(os.path.join(valid_path, filename), 'w') as stream: for data in map(lambda data: [data], data_to_serialize): serializers.serialize(output_format, data, indent=indent, stream=stream)
def handle_models(models, **options): output_format = options.get('format', 'json') indent = options.get('indent', None) show_traceback = options.get('traceback', False) propagate = options.get('propagate', True) use_natural_foreign_keys = options.get('use_natural_foreign_keys', False) use_natural_primary_keys = options.get('use_natural_primary_keys', False) # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if output_format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % output_format) try: serializers.get_serializer(output_format) except KeyError: raise CommandError("Unknown serialization format: %s" % output_format) objects = set() for model, model_slice in models: if isinstance(model_slice, str): objects.add(*list(model._default_manager.filter(pk__exact=model_slice))) elif not model_slice or type(model_slice) is list: items = model._default_manager.all() if model_slice and model_slice[0]: items = items.filter(pk__gte=model_slice[0]) if model_slice and model_slice[1]: items = items.filter(pk__lt=model_slice[1]) items = items.order_by(model._meta.pk.attname) objects.update(items) else: raise CommandError("Wrong slice: %s" % model_slice) def flatten(container): for i in container: if isinstance(i, list) or isinstance(i, tuple): for j in flatten(i): yield j else: yield i if propagate: root_objects = list(objects) collector = NestedObjects(using='default') collector.collect(root_objects) return serializers.serialize( output_format, flatten(collector.nested()), indent=indent, use_natural_foreign_keys=use_natural_foreign_keys, use_natural_primary_keys=use_natural_primary_keys, )
def test_unregister(self): """ Unregistering a serializer doesn't cause the registry to be repopulated. """ serializers.unregister_serializer("xml") serializers.register_serializer("json3", "django.core.serializers.json") public_formats = serializers.get_public_serializer_formats() self.assertNotIn("xml", public_formats) self.assertIn("json3", public_formats)
def loaddata(self, fixture_labels): connection = connections[self.using] # Keep a count of the installed objects and fixtures self.fixture_count = 0 self.loaded_object_count = 0 self.fixture_object_count = 0 self.models = set() self.serialization_formats = serializers.get_public_serializer_formats( ) # Django's test suite repeatedly tries to load initial_data fixtures # from apps that don't have any fixtures. Because disabling constraint # checks can be expensive on some database (especially MSSQL), bail # out early if no fixtures are found. for fixture_label in fixture_labels: if self.find_fixtures(fixture_label): break else: return self.objs_with_deferred_fields = [] with connection.constraint_checks_disabled(): for fixture_label in fixture_labels: self.load_label(fixture_label) for obj in self.objs_with_deferred_fields: obj.save_deferred_fields(using=self.using) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in self.models] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem installing fixtures: %s" % e, ) raise # If we found even one object in a fixture, we need to reset the # database sequences. if self.loaded_object_count > 0: self.reset_sequences(connection, self.models) if self.verbosity >= 1: if self.fixture_object_count == self.loaded_object_count: self.stdout.write( "Installed %d object(s) from %d fixture(s)" % (self.loaded_object_count, self.fixture_count)) else: self.stdout.write( "Installed %d object(s) (of %d) from %d fixture(s)" % (self.loaded_object_count, self.fixture_object_count, self.fixture_count))
def loadhelp(self, fixture_labels=['aristotle_help_files']): connection = connections[self.using] # Keep a count of the installed objects and fixtures self.fixture_count = 0 self.loaded_object_count = 0 self.fixture_object_count = 0 self.models = set() self.serialization_formats = serializers.get_public_serializer_formats() # Django's test suite repeatedly tries to load initial_data fixtures # from apps that don't have any fixtures. Because disabling constraint # checks can be expensive on some database (especially MSSQL), bail # out early if no fixtures are found. if not self.find_fixtures(): return with connection.constraint_checks_disabled(): self.load_label() # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in self.models] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem installing fixtures: %s" % e,) raise # If we found even one object in a fixture, we need to reset the # database sequences. if self.loaded_object_count > 0: sequence_sql = connection.ops.sequence_reset_sql(no_style(), self.models) if sequence_sql: if self.verbosity >= 2: self.stdout.write("Resetting sequences\n") with connection.cursor() as cursor: for line in sequence_sql: cursor.execute(line) if self.verbosity >= 1: if self.fixture_count == 0 and self.hide_empty: pass elif self.fixture_object_count == self.loaded_object_count: self.stdout.write("Installed %d object(s) from %d fixture(s)" % ( self.loaded_object_count, self.fixture_count) ) else: self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)" % ( self.loaded_object_count, self.fixture_object_count, self.fixture_count) )
def test_builtin_serializers(self): "Requesting a list of serializer formats popuates the registry" all_formats = set(serializers.get_serializer_formats()) public_formats = set(serializers.get_public_serializer_formats()) self.assertTrue('xml' in all_formats), self.assertTrue('xml' in public_formats) self.assertTrue('json2' in all_formats) self.assertTrue('json2' in public_formats) self.assertTrue('python' in all_formats) self.assertFalse('python' in public_formats)
def test_builtin_serializers(self): "Requesting a list of serializer formats popuates the registry" all_formats = set(serializers.get_serializer_formats()) public_formats = set(serializers.get_public_serializer_formats()) self.assertIn('xml', all_formats), self.assertIn('xml', public_formats) self.assertIn('json2', all_formats) self.assertIn('json2', public_formats) self.assertIn('python', all_formats) self.assertNotIn('python', public_formats)
def test_builtin_serializers(self): "Requesting a list of serializer formats popuates the registry" all_formats = set(serializers.get_serializer_formats()) public_formats = set(serializers.get_public_serializer_formats()) self.assertIn('xml', all_formats), self.assertIn('xml', public_formats) self.assertIn('json2', all_formats) self.assertIn('json2', public_formats) self.assertIn('python', all_formats) self.assertNotIn('python', public_formats)
def test_builtin_serializers(self): "Requesting a list of serializer formats populates the registry" all_formats = set(serializers.get_serializer_formats()) public_formats = set(serializers.get_public_serializer_formats()) self.assertIn("xml", all_formats), self.assertIn("xml", public_formats) self.assertIn("json2", all_formats) self.assertIn("json2", public_formats) self.assertIn("python", all_formats) self.assertNotIn("python", public_formats)
def loaddata(self, fixture_labels): connection = connections[self.using] # Keep a count of the installed objects and fixtures self.fixture_count = 0 self.loaded_object_count = 0 self.fixture_object_count = 0 self.models = set() self.serialization_formats = serializers.get_public_serializer_formats() self.compression_formats = { None: open, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: self.compression_formats['bz2'] = bz2.BZ2File with connection.constraint_checks_disabled(): for fixture_label in fixture_labels: self.load_label(fixture_label) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in self.models] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem installing fixtures: %s" % e,) raise # If we found even one object in a fixture, we need to reset the # database sequences. if self.loaded_object_count > 0: sequence_sql = connection.ops.sequence_reset_sql(no_style(), self.models) if sequence_sql: if self.verbosity >= 2: self.stdout.write("Resetting sequences\n") with connection.cursor() as cursor: for line in sequence_sql: cursor.execute(line) if self.verbosity >= 1: if self.fixture_count == 0 and self.hide_empty: pass elif self.fixture_object_count == self.loaded_object_count: self.stdout.write("Installed %d object(s) from %d fixture(s)" % (self.loaded_object_count, self.fixture_count)) else: self.stdout.write("Installed %d object(s) (of %d) from %d fixture(s)" % (self.loaded_object_count, self.fixture_object_count, self.fixture_count))
def handle(self, *fixture_labels, **options): formats = serializers.get_public_serializer_formats() app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(path) else: # It's a models.py module app_module_paths.append(app.__file__) app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths] for fixture_label in fixture_labels: if os.path.isabs(fixture_label): management.call_command('loaddata', *[fixture_label], **options) else: fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [''] fixture_dirs.sort() label_fixtures = [] found_in_fix_dir = '' try: for fixture_dir in fixture_dirs: filepath = os.path.join(fixture_dir, fixture_label) if os.path.exists(filepath): if os.path.isdir(filepath): for item in os.listdir(filepath): item_ext = os.path.splitext(item)[1] if item_ext and item_ext[1:] in formats: if fixture_label.split('/')[-1] in [item, item.split('.')[0]]: if found_in_fix_dir and found_in_fix_dir != fixture_dir: raise MultipleFixturesFoundError( "Found multiple files for %s\n" % fixture_label ) label_fixtures.append(os.path.join(filepath, item)) found_in_fix_dir = fixture_dir except MultipleFixturesFoundError, err: if options.get('traceback'): raise err else: raise CommandError("%s" % err) if len(label_fixtures): for label_fixture in label_fixtures: management.call_command('loaddata', label_fixture, **options) else: management.call_command('loaddata', fixture_label, **options)
def _handle_fragments(self, labels, paths): # IGNORE:W0142 dirs = [dir for dir in paths if os.path.isdir(dir)] fragments = [(dir, os.listdir(dir)) for dir in dirs] for dir, filenames in fragments: if not filenames: continue for filename in filenames: parts = filename.split('.') label, format = '.'.join(parts[:-1]), parts[-1] if 0 < len(labels) and not label in labels: if 1 < self.verbosity: print 'Skipping %s' % label continue if format in serializers.get_public_serializer_formats(): self._handle_fragment(dir, label, format) elif 0 < self.verbosity: print 'Unrecognized serialization format `%s` (%s in %s)' \ % (format, label, dir)
def handle(self, *app_labels, **options): self.options = options self.app_labels = app_labels self.relateds = [] format = self.options.get('format') indent = self.options.get('indent') show_traceback = self.options.get('traceback') use_natural_keys = self.options.get('use_natural_keys') self.excluded = self.options.get('excluded').split(',') # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) # Now collate the objects to be serialized. self.dump_list = InstancesList() for qs in self.get_querysets(): for item in qs: self.extract_fields(item) app_list = self.dump_list.model_list() obj_list = [] ordered = serializers.sort_dependencies(app_list.items()) explored = [] for item in ordered: if item not in explored: app_list = self.dump_list.get_items_by_model_path(item) obj_list.extend(app_list) explored.append(item) try: return serializers.serialize(format, obj_list, indent=indent, use_natural_keys=use_natural_keys) except Exception as e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def handle(self, *app_labels, **options): excluded_apps = [get_app(app_label) for app_label in options["exclude"] if "." not in app_label] excluded_models = [model.split(".") for model in options["exclude"] if "." in model] if len(app_labels) == 0: app_list = [app for app in get_apps() if app not in excluded_apps] else: app_list = [get_app(app_label) for app_label in app_labels] # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if options["format"] not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % options["format"]) try: serializers.get_serializer(options["format"]) except KeyError: raise CommandError("Unknown serialization format: %s" % options["format"]) # Output output_file = options.get("output") if output_file is None: output = sys.stdout else: output = open(output_file, "w") objects = [] models = [] for app in app_list: app_name = app.__name__.split(".")[-2] # assuming -1 is 'models' and -2 is name models.extend([model for model in get_models(app) if [app_name, model.__name__] not in excluded_models]) models = foreign_key_sort(models) for model in models: objects.extend(model._default_manager.all()) try: print serializers.serialize( options["format"], objects, indent=options["indent"], use_natural_keys=options["use_natural_keys"] ) except Exception, e: if options["traceback"]: raise raise CommandError("Unable to serialize database: %s" % e)
def handle(self, *app_labels, **options): self.options = options self.app_labels = app_labels self.relateds = [] format = self.options.get('format') indent = self.options.get('indent') show_traceback = self.options.get('traceback') use_natural_keys = self.options.get('use_natural_keys') self.excluded = self.options.get('excluded').split(',') # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) # Now collate the objects to be serialized. self.dump_list = InstancesList() for qs in self.get_querysets(): for item in qs: self.extract_fields(item) app_list = self.dump_list.model_list() obj_list = [] ordered = sort_dependencies(app_list.items()) explored = [] for item in ordered: if item not in explored: app_list = self.dump_list.get_items_by_model_path(item) obj_list.extend(app_list) explored.append(item) try: return serializers.serialize(format, obj_list, indent=indent, use_natural_keys=use_natural_keys) except Exception, e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def unloaddata(self, fixture_labels): connection = connections[self.using] self.fixture_count = 0 self.unloaded_object_count = 0 self.fixture_object_count = 0 self.serialization_formats = serializers.get_public_serializer_formats( ) self.compression_formats = { None: (open, "rb"), "gz": (gzip.GzipFile, "rb"), "zip": (SingleZipReader, "r"), "stdin": (lambda *args: sys.stdin, None), } if has_bz2: self.compression_formats["bz2"] = (bz2.BZ2File, "r") if has_lzma: self.compression_formats["lzma"] = (lzma.LZMAFile, "r") self.compression_formats["xz"] = (lzma.LZMAFile, "r") for fixture_label in fixture_labels: if self.find_fixtures(fixture_label): break else: return # here is the different bit... # just execute the single unload_label fn, instead of all the # trial-and-error that happens when actually loading a fixture self.class_deletion_record = collections.defaultdict(int) for fixture_label in fixture_labels: self.unload_label(fixture_label) if self.verbosity >= 1: self.stdout.write( "Unloaded %d object(s) (of %d) from %d fixture(s)" % ( self.unloaded_object_count, self.fixture_object_count, self.fixture_count, ))
def handle(self, *args, **options): using = options.get('database', DEFAULT_DB_ALIAS) connection = connections[using] def humanize(dirname): return "'%s'" % dirname if dirname else 'absolute path' schemas = options.get('schemas', []) for name in schemas: parts = name.split('.') format = parts[-1] if format in serializers.get_public_serializer_formats(): self.stdout.write("Loading '%s' schema...\n" % name) else: self.stderr.write( self.style.ERROR( "Problem installing schema '%s': %s is not a known serialization format.\n" % (name, format))) return try: full_path = name data = open(full_path) try: objects = serializers.deserialize(format, data) for obj in objects: if not options.get('replace', False): obj.object.pk = None obj.save(using=using) except Exception as err: if isinstance(err, (SystemExit, KeyboardInterrupt)): raise data.close() data.close() except Exception: self.stdout.write( "No %s schema '%s' in %s.\n" % (parts[-1], name, humanize(full_path)))
def handle(self, *args, **options): using = options.get('database', DEFAULT_DB_ALIAS) connection = connections[using] def humanize(dirname): return "'%s'" % dirname if dirname else 'absolute path' schemas = options.get('schemas', []) for name in schemas: parts = name.split('.') format = parts[-1] if format in serializers.get_public_serializer_formats(): self.stdout.write("Loading '%s' schema...\n" % name) else: self.stderr.write( self.style.ERROR( "Problem installing schema '%s': %s is not a known serialization format.\n" % (name, format))) return try: full_path = name data = open(full_path) try: objects = serializers.deserialize(format, data) for obj in objects: if not options.get('replace', False): obj.object.pk = None obj.save(using=using) except (SystemExit, KeyboardInterrupt): raise except Exception: data.close() data.close() except Exception: self.stdout.write( "No %s schema '%s' in %s.\n" % (parts[-1], name, humanize(full_path)))
def handle(self, *args, **options): using = options.get('database', DEFAULT_DB_ALIAS) connection = connections[using] show_traceback = options.get('traceback', False) format = options.get('format', 'json') if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) objects = [] if len(args) == 0: objects.extend(models.Schema.objects.all()) objects.extend(models.ParameterName.objects.all()) else: schemas = models.Schema.objects.filter(namespace__in=args) if len(schemas) == 0: raise CommandError('No schemas found') schema_set = set([s.namespace for s in schemas]) arg_set = set(args) skipped = arg_set - schema_set if len(skipped) > 0: sys.stderr.write('Schema not found: {0}\n'.format( \ ', '.join(skipped))) objects.extend(schemas) objects.extend( models.ParameterName.objects.filter( schema__namespace__in=args)) try: return serializers.serialize(format, objects, indent=4, use_natural_keys=True) except Exception, e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def handle(self, *args, **options): namespaces = options.get('namespaces', []) show_traceback = options.get('traceback', False) format = options.get('format', 'json') if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) objects = [] if not namespaces: objects.extend(models.Schema.objects.all()) objects.extend(models.ParameterName.objects.all()) else: schemas = models.Schema.objects.filter(namespace__in=namespaces) if not schemas: raise CommandError('No schemas found') schema_set = set(s.namespace for s in schemas) arg_set = set(args) skipped = arg_set - schema_set if skipped: sys.stderr.write('Schema not found: {0}\n'.format( ', '.join(skipped))) objects.extend(schemas) objects.extend( models.ParameterName.objects.filter( schema__namespace__in=args)) try: return serializers.serialize(format, objects, indent=4, use_natural_foreign_keys=True) except Exception as e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def handle(self, *labels, **options): meetingname = labels[0] schedname = labels[1] from ietf.meeting.helpers import get_meeting, get_schedule format = options.get('format', 'json') indent = options.get('indent', 2) meeting = get_meeting(meetingname) schedule = get_schedule(meeting, schedname) assignments = schedule.assignments.all() # cribbed from django/core/management/commands/dumpdata.py # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) return serializers.serialize(format, assignments, indent=indent, use_natural_keys=True)
def handle(self, *app_labels, **options): from django.db.models import get_app, get_apps, get_models, get_model format = options.get('format', 'json') indent = options.get('indent', None) using = options.get('database', DEFAULT_DB_ALIAS) connection = connections[using] excludes = options.get('exclude', []) show_traceback = options.get('traceback', False) use_natural_keys = options.get('use_natural_keys', False) excluded_apps = set() excluded_models = set() for exclude in excludes: if '.' in exclude: app_label, model_name = exclude.split('.', 1) model_obj = get_model(app_label, model_name) if not model_obj: raise CommandError('Unknown model in excludes: %s' % exclude) excluded_models.add(model_obj) else: try: app_obj = get_app(exclude) excluded_apps.add(app_obj) except ImproperlyConfigured: raise CommandError('Unknown app in excludes: %s' % exclude) if len(app_labels) == 0: app_list = SortedDict( (app, None) for app in get_apps() if app not in excluded_apps) else: app_list = SortedDict() for label in app_labels: try: app_label, model_label = label.split('.') try: app = get_app(app_label) except ImproperlyConfigured: raise CommandError("Unknown application: %s" % app_label) if app in excluded_apps: continue model = get_model(app_label, model_label) if model is None: raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) if app in app_list.keys(): if app_list[app] and model not in app_list[app]: app_list[app].append(model) else: app_list[app] = [model] except ValueError: # This is just an app - no model qualifier app_label = label try: app = get_app(app_label) except ImproperlyConfigured: raise CommandError("Unknown application: %s" % app_label) if app in excluded_apps: continue app_list[app] = None # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) # Now collate the objects to be serialized. objects = [] for model in sort_dependencies(app_list.items()): if model in excluded_models: continue if not model._meta.proxy and router.allow_syncdb(using, model): objects.extend(model._default_manager.using(using).all()) try: return serializers.serialize(format, objects, indent=indent, use_natural_keys=use_natural_keys) except Exception, e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def tables_used_by_fixtures(fixture_labels, using=DEFAULT_DB_ALIAS): """Act like Django's stock loaddata command, but, instead of loading data, return an iterable of the names of the tables into which data would be loaded.""" # Keep a count of the installed objects and fixtures fixture_count = 0 loaded_object_count = 0 fixture_object_count = 0 tables = set() class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len(self.namelist()) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) compression_types = { None: file, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: compression_types['bz2'] = bz2.BZ2File app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(path) else: # It's a models.py module app_module_paths.append(app.__file__) app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = list(compression_types.keys()) if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if not formats: # stderr.write(style.ERROR("Problem installing fixture '%s': %s is # not a known serialization format.\n" % (fixture_name, format))) return set() if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list(settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: # stdout.write("Checking %s for fixtures...\n" % # humanize(fixture_dir)) label_found = False for combo in product([using, None], formats, compression_formats): database, format, compression_format = combo file_name = '.'.join( p for p in [ fixture_name, database, format, compression_format ] if p ) # stdout.write("Trying %s for %s fixture '%s'...\n" % \ # (humanize(fixture_dir), file_name, fixture_name)) full_path = os.path.join(fixture_dir, file_name) open_method = compression_types[compression_format] try: fixture = open_method(full_path, 'r') if label_found: fixture.close() # stderr.write(style.ERROR("Multiple fixtures named # '%s' in %s. Aborting.\n" % (fixture_name, # humanize(fixture_dir)))) return set() else: fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 # stdout.write("Installing %s fixture '%s' from %s.\n" # % (format, fixture_name, humanize(fixture_dir))) try: objects = serializers.deserialize(format, fixture, using=using) for obj in objects: objects_in_fixture += 1 if router.allow_syncdb(using, obj.object.__class__): loaded_objects_in_fixture += 1 tables.add( obj.object.__class__._meta.db_table) loaded_object_count += loaded_objects_in_fixture fixture_object_count += objects_in_fixture label_found = True except (SystemExit, KeyboardInterrupt): raise except Exception: fixture.close() # stderr.write( style.ERROR("Problem installing # fixture '%s': %s\n" % (full_path, ''.join(tra # ceback.format_exception(sys.exc_type, # sys.exc_value, sys.exc_traceback))))) return set() fixture.close() # If the fixture we loaded contains 0 objects, assume that an # error was encountered during fixture loading. if objects_in_fixture == 0: # stderr.write( style.ERROR("No fixture data found # for '%s'. (File format may be invalid.)\n" % # (fixture_name))) return set() except Exception: # stdout.write("No %s fixture '%s' in %s.\n" % \ (format, # fixture_name, humanize(fixture_dir))) pass return tables
def handle_noargs(self, **options): """ Generates a dump of Django CMS database. """ format = options.get('format') indent = options.get('indent') use_natural_keys = options.get('use_natural_keys') show_traceback = options.get('traceback') # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise base.CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise base.CommandError("Unknown serialization format: %s" % format) pages = cms_models.Page.objects.filter(published=True) placeholders = cms_models.Placeholder.objects.filter(page__in=pages) blog_entries = blog_models.Entry.published.all() blog_placeholders = cms_models.Placeholder.objects.filter( entry__in=blog_entries) plugins = cms_models.CMSPlugin.objects.filter( Q(placeholder__in=placeholders) | Q(placeholder__in=blog_placeholders)) def plugin_objects(): for plugin in plugins: instance = plugin.get_plugin_instance()[0] if instance is None: raise base.CommandError( "Invalid plugin instance: %s from placeholder %s" % (plugin.pk, plugin.placeholder.pk)) yield instance objects = itertools.chain( sites_models.Site.objects.all(), contenttypes_models.ContentType.objects.all(), placeholders, pages, cms_models.Title.objects.filter(page__in=pages), blog_placeholders, blog_entries, blog_models.EntryTitle.objects.filter(entry__in=blog_entries), plugins, plugin_objects(), filer_models.Folder.objects.all(), filer_models.File.objects.non_polymorphic().filter(is_public=True), filer_models.Image.objects.non_polymorphic().filter( is_public=True), filer_image_models.ThumbnailOption.objects.all(), snippet_models.Snippet.objects.all(), ) try: return serializers.serialize(format, objects, indent=indent, use_natural_keys=use_natural_keys) except Exception, e: if show_traceback: raise raise base.CommandError("Unable to serialize database: %s" % e)
def export_choices(): fmts = serializers.get_public_serializer_formats() return zip(fmts,fmts)
def import_from(self, infile, maxsize=10000000): errors = [] results = [] filetype = infile.content_type filename = infile.name raw = infile.read() # filelen = len(raw) # if filelen > maxsize: # errors.append(_('Import too large, must be smaller than %i bytes.' % maxsize )) format = os.path.splitext(filename)[1] if format and format.startswith('.'): format = format[1:] if not format: errors.append(_('Could not parse format from filename: %s') % filename) if format == 'zip': zf = zipfile.ZipFile(StringIO(raw), 'r') files = zf.namelist() image_dir = config_value('PRODUCT', 'IMAGE_DIR') other_image_dir = None export_file = None if 'VARS' in files: config = zf.read('VARS') lines = [line.split('=') for line in config.split('\n')] for key, val in lines: if key == 'PRODUCT.IMAGE_DIR': other_image_dir = val elif key == 'EXPORT_FILE': export_file = val if other_image_dir is None or export_file is None: errors.append(_('Bad VARS file in import zipfile.')) else: # save out all the files which start with other_image_dr rename = image_dir == other_image_dir for f in files: if f.startswith(other_image_dir): buf = zf.read(f) if rename: f = f[len(other_image_dir):] if f[0] in ('/', '\\'): f = f[1:] f = os.path.join(settings.MEDIA_ROOT, image_dir, f) outf = open(f, 'w') outf.write(buf) outf.close() results.append('Imported image: %s' % f) infile = zf.read(export_file) zf.close() format = os.path.splitext(export_file)[1] if format and format.startswith('.'): format = format[1:] if not format: errors.append(_('Could not parse format from filename: %s') % filename) else: raw = infile else: errors.append(_('Missing VARS in import zipfile.')) else: raw = StringIO(str(raw)) if not format in serializers.get_public_serializer_formats(): errors.append(_('Unknown file format: %s') % format) if not errors: from django.db import connection, transaction transaction.commit() transaction.enter_transaction_management() try: ct = 0 models = set() for obj in serializers.deserialize(format, raw): obj.save() models.add(obj.object.__class__) ct += 1 if ct>0: style=no_style() sequence_sql = connection.ops.sequence_reset_sql(style, models) if sequence_sql: cursor = connection.cursor() for line in sequence_sql: cursor.execute(line) results.append(_('Added %(count)i objects from %(filename)s') % {'count': ct, 'filename': filename}) transaction.commit() #label_found = True except Exception, e: #fixture.close() errors.append(_("Problem installing fixture '%(filename)s': %(error_msg)s\n") % {'filename': filename, 'error_msg': str(e)}) errors.append("Raw: %s" % raw) transaction.rollback() transaction.leave_transaction_management() else: transaction.leave_transaction_management()
def handle(self, *app_labels, **options): from django.db.models import get_app, get_apps, get_model format = options.get('format') indent = options.get('indent') using = options.get('database') excludes = options.get('exclude') show_traceback = options.get('traceback') use_natural_keys = options.get('use_natural_keys') use_base_manager = options.get('use_base_manager') pks = options.get('primary_keys') if pks: primary_keys = pks.split(',') else: primary_keys = [] excluded_apps = set() excluded_models = set() for exclude in excludes: if '.' in exclude: app_label, model_name = exclude.split('.', 1) model_obj = get_model(app_label, model_name) if not model_obj: raise CommandError('Unknown model in excludes: %s' % exclude) excluded_models.add(model_obj) else: try: app_obj = get_app(exclude) excluded_apps.add(app_obj) except ImproperlyConfigured: raise CommandError('Unknown app in excludes: %s' % exclude) if len(app_labels) == 0: if primary_keys: raise CommandError("You can only use --pks option with one model") app_list = OrderedDict((app, None) for app in get_apps() if app not in excluded_apps) else: if len(app_labels) > 1 and primary_keys: raise CommandError("You can only use --pks option with one model") app_list = OrderedDict() for label in app_labels: try: app_label, model_label = label.split('.') try: app = get_app(app_label) except ImproperlyConfigured: raise CommandError("Unknown application: %s" % app_label) if app in excluded_apps: continue model = get_model(app_label, model_label) if model is None: raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) if app in app_list.keys(): if app_list[app] and model not in app_list[app]: app_list[app].append(model) else: app_list[app] = [model] except ValueError: if primary_keys: raise CommandError("You can only use --pks option with one model") # This is just an app - no model qualifier app_label = label try: app = get_app(app_label) except ImproperlyConfigured: raise CommandError("Unknown application: %s" % app_label) if app in excluded_apps: continue app_list[app] = None # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): raise CommandError("Unknown serialization format: %s" % format) try: serializers.get_serializer(format) except KeyError: raise CommandError("Unknown serialization format: %s" % format) def get_objects(): # Collate the objects to be serialized. for model in sort_dependencies(app_list.items()): if model in excluded_models: continue if not model._meta.proxy and router.allow_syncdb(using, model): if use_base_manager: objects = model._base_manager else: objects = model._default_manager queryset = objects.using(using).order_by(model._meta.pk.name) if primary_keys: queryset = queryset.filter(pk__in=primary_keys) for obj in queryset.iterator(): yield obj try: self.stdout.ending = None serializers.serialize(format, get_objects(), indent=indent, use_natural_keys=use_natural_keys, stream=self.stdout) except Exception as e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def loaddata(self, fixture_labels): connection = connections[self.using] # Keep a count of the installed objects and fixtures self.fixture_count = 0 self.loaded_object_count = 0 self.fixture_object_count = 0 self.models = set() self.serialization_formats = serializers.get_public_serializer_formats( ) # Forcing binary mode may be revisited after dropping Python 2 support (see #22399) self.compression_formats = { None: (open, 'rb'), 'gz': (gzip.GzipFile, 'rb'), 'zip': (SingleZipReader, 'r'), 'stdin': (lambda *args: sys.stdin, None), } if has_bz2: self.compression_formats['bz2'] = (bz2.BZ2File, 'r') # Django's tests suite repeatedly tries to load initial_data fixtures # from apps that don't have any fixtures. Because disabling constraint # checks can be expensive on some database (especially MSSQL), bail # out early if no fixtures are found. for fixture_label in fixture_labels: if self.find_fixtures(fixture_label): break else: return with connection.constraint_checks_disabled(): self.objs_with_deferred_fields = [] for fixture_label in fixture_labels: self.load_label(fixture_label) for obj in self.objs_with_deferred_fields: obj.save_deferred_fields(using=self.using) # Since we disabled constraint checks, we must manually check for # any invalid keys that might have been added table_names = [model._meta.db_table for model in self.models] try: connection.check_constraints(table_names=table_names) except Exception as e: e.args = ("Problem installing fixtures: %s" % e, ) raise # If we found even one object in a fixture, we need to reset the # database sequences. if self.loaded_object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( no_style(), self.models) if sequence_sql: if self.verbosity >= 2: self.stdout.write('Resetting sequences') with connection.cursor() as cursor: for line in sequence_sql: cursor.execute(line) if self.verbosity >= 1: if self.fixture_object_count == self.loaded_object_count: self.stdout.write( "Installed %d object(s) from %d fixture(s)" % (self.loaded_object_count, self.fixture_count)) else: self.stdout.write( "Installed %d object(s) (of %d) from %d fixture(s)" % (self.loaded_object_count, self.fixture_object_count, self.fixture_count))
def handle(self, *app_labels, **options): format = options['format'] indent = options['indent'] using = options['database'] excludes = options['exclude'] output = options['output'] show_traceback = options['traceback'] use_natural_foreign_keys = options['use_natural_foreign_keys'] use_natural_primary_keys = options['use_natural_primary_keys'] use_base_manager = options['use_base_manager'] pks = options['primary_keys'] if pks: primary_keys = [pk.strip() for pk in pks.split(',')] else: primary_keys = [] excluded_models, excluded_apps = parse_apps_and_model_labels(excludes) if not app_labels: if primary_keys: raise CommandError("You can only use --pks option with one model") app_list = dict.fromkeys( app_config for app_config in apps.get_app_configs() if app_config.models_module is not None and app_config not in excluded_apps ) else: if len(app_labels) > 1 and primary_keys: raise CommandError("You can only use --pks option with one model") app_list = {} for label in app_labels: try: app_label, model_label = label.split('.') try: app_config = apps.get_app_config(app_label) except LookupError as e: raise CommandError(str(e)) if app_config.models_module is None or app_config in excluded_apps: continue try: model = app_config.get_model(model_label) except LookupError: raise CommandError("Unknown model: %s.%s" % (app_label, model_label)) app_list_value = app_list.setdefault(app_config, []) # We may have previously seen an "all-models" request for # this app (no model qualifier was given). In this case # there is no need adding specific models to the list. if app_list_value is not None: if model not in app_list_value: app_list_value.append(model) except ValueError: if primary_keys: raise CommandError("You can only use --pks option with one model") # This is just an app - no model qualifier app_label = label try: app_config = apps.get_app_config(app_label) except LookupError as e: raise CommandError(str(e)) if app_config.models_module is None or app_config in excluded_apps: continue app_list[app_config] = None # Check that the serialization format exists; this is a shortcut to # avoid collating all the objects and _then_ failing. if format not in serializers.get_public_serializer_formats(): try: serializers.get_serializer(format) except serializers.SerializerDoesNotExist: pass raise CommandError("Unknown serialization format: %s" % format) def get_objects(count_only=False): """ Collate the objects to be serialized. If count_only is True, just count the number of objects to be serialized. """ models = serializers.sort_dependencies(app_list.items()) for model in models: if model in excluded_models: continue if model._meta.proxy and model._meta.proxy_for_model not in models: warnings.warn( "%s is a proxy model and won't be serialized." % model._meta.label, category=ProxyModelWarning, ) if not model._meta.proxy and router.allow_migrate_model(using, model): if use_base_manager: objects = model._base_manager else: objects = model._default_manager queryset = objects.using(using).order_by(model._meta.pk.name) if primary_keys: queryset = queryset.filter(pk__in=primary_keys) if count_only: yield queryset.order_by().count() else: yield from queryset.iterator() try: self.stdout.ending = None progress_output = None object_count = 0 # If dumpdata is outputting to stdout, there is no way to display progress if output and self.stdout.isatty() and options['verbosity'] > 0: progress_output = self.stdout object_count = sum(get_objects(count_only=True)) stream = open(output, 'w') if output else None try: serializers.serialize( format, get_objects(), indent=indent, use_natural_foreign_keys=use_natural_foreign_keys, use_natural_primary_keys=use_natural_primary_keys, stream=stream or self.stdout, progress_output=progress_output, object_count=object_count, ) finally: if stream: stream.close() except Exception as e: if show_traceback: raise raise CommandError("Unable to serialize database: %s" % e)
def handle(self, *fixture_labels, **options): using = options.get('database', DEFAULT_DB_ALIAS) connection = connections[using] self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # commit is a stealth option - it isn't really useful as # a command line option, but it can be useful when invoking # loaddata from within another script. # If commit=True, loaddata will use its own transaction; # if commit=False, the data load SQL will become part of # the transaction in place when loaddata was invoked. commit = options.get('commit', True) # Keep a count of the installed objects and fixtures fixture_count = 0 loaded_object_count = 0 fixture_object_count = 0 models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. if commit: transaction.commit_unless_managed(using=using) transaction.enter_transaction_management(using=using) transaction.managed(True, using=using) class SingleZipReader(zipfile.ZipFile): def __init__(self, *args, **kwargs): zipfile.ZipFile.__init__(self, *args, **kwargs) if settings.DEBUG: assert len( self.namelist() ) == 1, "Zip-compressed fixtures must contain only one file." def read(self): return zipfile.ZipFile.read(self, self.namelist()[0]) compression_types = { None: file, 'gz': gzip.GzipFile, 'zip': SingleZipReader } if has_bz2: compression_types['bz2'] = bz2.BZ2File app_module_paths = [] for app in get_apps(): if hasattr(app, '__path__'): # It's a 'models/' subpackage for path in app.__path__: app_module_paths.append(path) else: # It's a models.py module app_module_paths.append(app.__file__) app_fixtures = [ os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths ] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) > 1 and parts[-1] in compression_types: compression_formats = [parts[-1]] parts = parts[:-1] else: compression_formats = compression_types.keys() if len(parts) == 1: fixture_name = parts[0] formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity > 1: self.stdout.write("Loading '%s' fixtures...\n" % fixture_name) else: self.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s is not a known serialization format.\n" % (fixture_name, format))) if commit: transaction.rollback(using=using) transaction.leave_transaction_management(using=using) return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity > 1: self.stdout.write("Checking %s for fixtures...\n" % humanize(fixture_dir)) label_found = False for combo in product([using, None], formats, compression_formats): database, format, compression_format = combo file_name = '.'.join( p for p in [fixture_name, database, format, compression_format] if p) if verbosity > 1: self.stdout.write("Trying %s for %s fixture '%s'...\n" % \ (humanize(fixture_dir), file_name, fixture_name)) full_path = os.path.join(fixture_dir, file_name) open_method = compression_types[compression_format] try: fixture = open_method(full_path, 'r') if label_found: fixture.close() self.stderr.write( self.style.ERROR( "Multiple fixtures named '%s' in %s. Aborting.\n" % (fixture_name, humanize(fixture_dir)))) if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) return else: fixture_count += 1 objects_in_fixture = 0 loaded_objects_in_fixture = 0 if verbosity > 0: self.stdout.write("Installing %s fixture '%s' from %s.\n" % \ (format, fixture_name, humanize(fixture_dir))) try: objects = serializers.deserialize(format, fixture, using=using) for obj in objects: objects_in_fixture += 1 if router.allow_syncdb( using, obj.object.__class__): loaded_objects_in_fixture += 1 models.add(obj.object.__class__) obj.save(using=using) loaded_object_count += loaded_objects_in_fixture fixture_object_count += objects_in_fixture label_found = True except (SystemExit, KeyboardInterrupt): raise except Exception: import traceback fixture.close() if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) if show_traceback: traceback.print_exc() else: self.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s\n" % (full_path, ''.join( traceback.format_exception( sys.exc_type, sys.exc_value, sys.exc_traceback))))) return fixture.close() # If the fixture we loaded contains 0 objects, assume that an # error was encountered during fixture loading. if objects_in_fixture == 0: self.stderr.write( self.style.ERROR( "No fixture data found for '%s'. (File format may be invalid.)\n" % (fixture_name))) if commit: transaction.rollback(using=using) transaction.leave_transaction_management( using=using) return except Exception, e: if verbosity > 1: self.stdout.write("No %s fixture '%s' in %s.\n" % \ (format, fixture_name, humanize(fixture_dir)))
def handle(self, *fixture_labels, **options): """ Main method of a Django command """ from django.db.models import get_apps from django.core import serializers from django.db import connection, transaction from django.conf import settings self.style = no_style() verbosity = int(options.get('verbosity', 1)) show_traceback = options.get('traceback', False) # Keep a count of the installed objects and fixtures fixture_count = 0 object_count = 0 objects_per_fixture = [] models = set() humanize = lambda dirname: dirname and "'%s'" % dirname or 'absolute path' # Get a cursor (even though we don't need one yet). This has # the side effect of initializing the test database (if # it isn't already initialized). cursor = connection.cursor() # Start transaction management. All fixtures are installed in a # single transaction to ensure that all references are resolved. transaction.commit_unless_managed() transaction.enter_transaction_management() transaction.managed(True) app_fixtures = [os.path.join(os.path.dirname(app.__file__), 'fixtures') \ for app in get_apps()] for fixture_label in fixture_labels: parts = fixture_label.split('.') if len(parts) == 1: fixture_name = fixture_label formats = serializers.get_public_serializer_formats() else: fixture_name, format = '.'.join(parts[:-1]), parts[-1] if format in serializers.get_public_serializer_formats(): formats = [format] else: formats = [] if formats: if verbosity > 1: print "Loading '%s' fixtures..." % fixture_name else: sys.stderr.write( self.style.ERROR("Problem installing fixture '%s': %s is not a known " + \ "serialization format." % (fixture_name, format)) ) transaction.rollback() transaction.leave_transaction_management() return if os.path.isabs(fixture_name): fixture_dirs = [fixture_name] else: fixture_dirs = app_fixtures + list( settings.FIXTURE_DIRS) + [''] for fixture_dir in fixture_dirs: if verbosity > 1: print "Checking %s for fixtures..." % humanize(fixture_dir) label_found = False for format in formats: serializer = serializers.get_serializer(format) if verbosity > 1: print "Trying %s for %s fixture '%s'..." % \ (humanize(fixture_dir), format, fixture_name) try: full_path = os.path.join( fixture_dir, '.'.join([fixture_name, format])) fixture = open(full_path, 'r') if label_found: fixture.close() print self.style.ERROR( "Multiple fixtures named '%s' in %s. Aborting." % (fixture_name, humanize(fixture_dir))) transaction.rollback() transaction.leave_transaction_management() return else: fixture_count += 1 objects_per_fixture.append(0) if verbosity > 0: print "Installing %s fixture '%s' from %s." % \ (format, fixture_name, humanize(fixture_dir)) try: objects_to_keep = {} objects = serializers.deserialize( format, fixture) for obj in objects: object_count += 1 objects_per_fixture[-1] += 1 class_ = obj.object.__class__ if not class_ in objects_to_keep: objects_to_keep[class_] = set() objects_to_keep[class_].add(obj.object) models.add(class_) obj.save() self.remove_objects_not_in( objects_to_keep, verbosity) label_found = True except (SystemExit, KeyboardInterrupt): raise except Exception: import traceback fixture.close() transaction.rollback() transaction.leave_transaction_management() if show_traceback: traceback.print_exc() else: sys.stderr.write( self.style.ERROR( "Problem installing fixture '%s': %s\n" % (full_path, traceback.format_exc()))) return fixture.close() except: if verbosity > 1: print "No %s fixture '%s' in %s." % \ (format, fixture_name, humanize(fixture_dir)) # If any of the fixtures we loaded contain 0 objects, assume that an # error was encountered during fixture loading. if 0 in objects_per_fixture: sys.stderr.write( self.style.ERROR( "No fixture data found for '%s'. (File format may be invalid.)" % (fixture_name))) transaction.rollback() transaction.leave_transaction_management() return # If we found even one object in a fixture, we need to reset the # database sequences. if object_count > 0: sequence_sql = connection.ops.sequence_reset_sql( self.style, models) if sequence_sql: if verbosity > 1: print "Resetting sequences" for line in sequence_sql: cursor.execute(line) transaction.commit() transaction.leave_transaction_management() if object_count == 0: if verbosity > 1: print "No fixtures found." else: if verbosity > 0: print "Installed %d object(s) from %d fixture(s)" % ( object_count, fixture_count) # Close the DB connection. This is required as a workaround for an # edge case in MySQL: if the same connection is used to # create tables, load data, and query, the query can return # incorrect results. See Django #7572, MySQL #37735. connection.close()