def dump_data(request,appname):
app_list = SortedDict()
try:
if request.POST:
for appname in request.POST.getlist('apps'):
app = get_app(appname)
app_list[app] = None
appname = 'choices'
else:
app = get_app(appname)
app_list[app] = None
except ImproperlyConfigured:
if appname == 'all':
for app in get_apps():
app_list[app] = None
if(len(app_list) > 0):
objects = []
for model in sort_dependencies(app_list.items()):
if not model._meta.proxy and router.allow_syncdb(DEFAULT_DB_ALIAS, model):
objects.extend(model._default_manager.using(DEFAULT_DB_ALIAS).all())
serializers.get_serializer('json')
json = serializers.serialize('json', objects, indent=2,use_natural_keys=True)
response = HttpResponse(json, mimetype='application/json');
response['Content-Disposition'] = 'attachment; filename=%s_%s_fixture.json' % (date.today().__str__(),appname)
return response
return render_to_response('diagnostic/dumpdata.html',context_instance=RequestContext(request))
def get_dependencies(models):
"""
Function to help finding model dependencies.
"""
all_models = set()
to_process = models
while len(to_process):
new_to_process = []
for model in to_process:
if model in all_models:
continue
all_models.add(model)
for field in model._meta.fields:
if hasattr(field.rel, 'to'):
new_to_process.append(field.rel.to)
for field in model._meta.many_to_many:
new_to_process.append(field.rel.to)
to_process = new_to_process
return dumpdata.sort_dependencies([(None, (model,)) for model in all_models])
def get_dependencies(models):
"""
Function to help finding model dependencies.
"""
all_models = set()
to_process = models
while len(to_process):
new_to_process = []
for model in to_process:
if model in all_models:
continue
all_models.add(model)
for field in model._meta.fields:
if hasattr(field.rel, 'to'):
new_to_process.append(field.rel.to)
for field in model._meta.many_to_many:
new_to_process.append(field.rel.to)
to_process = new_to_process
return dumpdata.sort_dependencies([(None, (model, ))
for model in all_models])
def get_objects():
for model in sort_dependencies(app_list):
if (not model._meta.proxy and model._meta.managed and
router.allow_migrate(self.connection.alias, model)):
queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
for obj in queryset.iterator():
yield obj
def test_dependency_sorting_5(self):
sorted_deps = sort_dependencies(
[('fixtures_regress', [Person, Book, Store])]
)
self.assertEqual(
sorted_deps,
[Store, Person, Book]
)
def test_dependency_sorting_dangling(self):
sorted_deps = sort_dependencies(
[('fixtures_regress', [Person, Circle1, Store, Book])]
)
self.assertEqual(
sorted_deps,
[Circle1, Store, Person, Book]
)
def test_dependency_sorting_normal(self):
sorted_deps = sort_dependencies(
[('fixtures_regress', [Person, ExternalDependency, Book])]
)
self.assertEqual(
sorted_deps,
[Person, Book, ExternalDependency]
)
def get_objects():
for model in sort_dependencies(app_list):
if not model._meta.proxy and model._meta.managed and router.allow_migrate(
self.connection.alias, model):
queryset = model._default_manager.using(
self.connection.alias).order_by(model._meta.pk.name)
for obj in queryset.iterator():
yield obj
def test_dependency_sorting_5(self):
sorted_deps = sort_dependencies(
[('fixtures_regress', [Person, Book, Store])]
)
self.assertEqual(
sorted_deps,
[Store, Person, Book]
)
def test_dependency_sorting_dangling(self):
sorted_deps = sort_dependencies(
[('fixtures_regress', [Person, Circle1, Store, Book])]
)
self.assertEqual(
sorted_deps,
[Circle1, Store, Person, Book]
)
def test_dependency_sorting_normal(self):
sorted_deps = sort_dependencies(
[('fixtures_regress', [Person, ExternalDependency, Book])]
)
self.assertEqual(
sorted_deps,
[Person, Book, ExternalDependency]
)
def test_dependency_sorting(self):
"""
Now lets check the dependency sorting explicitly
It doesn't matter what order you mention the models
Store *must* be serialized before then Person, and both
must be serialized before Book.
"""
sorted_deps = sort_dependencies([("fixtures_regress", [Book, Person, Store])])
self.assertEqual(sorted_deps, [Store, Person, Book])
def sort_relation_dependencies(app_list):
models = sort_dependencies(app_list.items())
models.reverse()
#print "(before) #models: %d" % len(models)
model_list, visited = [], []
while models:
model = models.pop(0)
_sort_relation_order(model, model_list, visited)
#print "(after) #models: %d" % len(model_list)
return model_list
def sort_relation_dependencies(app_list):
models = sort_dependencies(app_list.items())
models.reverse()
#print "(before) #models: %d" % len(models)
model_list, visited = [], []
while models:
model = models.pop(0)
_sort_relation_order(model, model_list, visited)
#print "(after) #models: %d" % len(model_list)
return model_list
def test_dependency_sorting(self):
"""
Now lets check the dependency sorting explicitly
It doesn't matter what order you mention the models
Store *must* be serialized before then Person, and both
must be serialized before Book.
"""
sorted_deps = sort_dependencies([('fixtures_regress',
[Book, Person, Store])])
self.assertEqual(sorted_deps, [Store, Person, Book])
def test_dependency_sorting_m2m_simple(self):
"""
M2M relations without explicit through models SHOULD count as dependencies
Regression test for bugs that could be caused by flawed fixes to
#14226, namely if M2M checks are removed from sort_dependencies
altogether.
"""
sorted_deps = sort_dependencies([('fixtures_regress',
[M2MSimpleA, M2MSimpleB])])
self.assertEqual(sorted_deps, [M2MSimpleB, M2MSimpleA])
def test_dependency_sorting_m2m_complex(self):
"""
M2M relations with explicit through models should NOT count as
dependencies. The through model itself will have dependencies, though.
"""
sorted_deps = sort_dependencies([
('fixtures_regress', [M2MComplexA, M2MComplexB, M2MThroughAB])
])
# Order between M2MComplexA and M2MComplexB doesn't matter. The through
# model has dependencies to them though, so it should come last.
self.assertEqual(sorted_deps[-1], M2MThroughAB)
def test_dependency_sorting_m2m_complex(self):
"""
M2M relations with explicit through models should NOT count as
dependencies. The through model itself will have dependencies, though.
"""
sorted_deps = sort_dependencies(
[('fixtures_regress', [M2MComplexA, M2MComplexB, M2MThroughAB])]
)
# Order between M2MComplexA and M2MComplexB doesn't matter. The through
# model has dependencies to them though, so it should come last.
self.assertEqual(sorted_deps[-1], M2MThroughAB)
def test_dependency_sorting_m2m_simple(self):
"""
M2M relations without explicit through models SHOULD count as dependencies
Regression test for bugs that could be caused by flawed fixes to
#14226, namely if M2M checks are removed from sort_dependencies
altogether.
"""
sorted_deps = sort_dependencies(
[('fixtures_regress', [M2MSimpleA, M2MSimpleB])]
)
self.assertEqual(sorted_deps, [M2MSimpleB, M2MSimpleA])
def handle(self, *args, **options):
from django.db.models import get_app
#TODO : Check export mode
db.reset_queries()
sourceDatabase = options.get('sourceDatabase')
dataLimit = options.get('dataLimit')
app = get_app(options.get('app'))
con = connections[sourceDatabase]
if con.vendor != 'oracle':
print "Source database has to be oracle."
return
user = settings.DATABASES[sourceDatabase]['USER']
passwd = settings.DATABASES[sourceDatabase]['PASSWORD']
host = settings.DATABASES[sourceDatabase]['HOST']
port = settings.DATABASES[sourceDatabase]['PORT']
name = settings.DATABASES[sourceDatabase]['NAME']
app_list = SortedDict((app, None) for app in [app])
tables = []
sorted = sort_dependencies(app_list.items())
lastObjectName = sorted[-1].__name__
filename = lastObjectName + ".postgresql_psycopg2.sql"
chemblSQLPath = os.path.join(os.path.dirname(app.__file__),'sql', filename)
location = chemblSQLPath
oracleHome = os.environ['ORACLE_HOME']
if options.get('dumpfile'):
if not options.get('dumpfile').endswith('.sql'):
location = os.path.join(options.get('dumpfile'), filename)
else:
location = options.get('dumpfile')
for model in reversed(sorted):
if not model._meta.managed:
continue
tables.append(model._meta.db_table)
print self.confTemplate % (oracleHome, host, name, port, user, passwd, user, " ".join(tables), dataLimit, location)
if location != chemblSQLPath:
print "different! location = " + location + ", chemblSQLPath = " + chemblSQLPath
f = open(location, 'w')
f.close()
os.symlink(location, chemblSQLPath)
#-----------------------------------------------------------------------------------------------------------------------
def get_objects():
# Collate the objects to be serialized.
for model in sort_dependencies(app_list.items()):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_syncdb(using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = QuerySet(model).all()
queryset = objects.using(using).order_by(model._meta.pk.name)
if primary_keys:
queryset = queryset.filter(pk__in=primary_keys)
for obj in queryset.iterator():
yield obj
def get_objects():
"""
Collate the objects for serialization, taken from dumpdata command
and adjusted to sanitize User password hashes
"""
# Collate the objects to be serialized.
for model in sort_dependencies(app_list.items()):
if not model._meta.proxy and router.allow_migrate(DEFAULT_DB_ALIAS, model):
objects = model._base_manager
queryset = objects.using(DEFAULT_DB_ALIAS).order_by(model._meta.pk.name)
for obj in queryset.iterator():
# Sanitize user objects
if model._meta.model_name == 'user':
obj.set_unusable_password()
yield obj
def get_objects():
# Collate the objects to be serialized.
for model in sort_dependencies(app_list.items()):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_syncdb(using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = QuerySet(model).all()
queryset = objects.using(using).order_by(
model._meta.pk.name)
if primary_keys:
queryset = queryset.filter(pk__in=primary_keys)
for obj in queryset.iterator():
yield obj
def test_dependency_sorting_m2m_complex_circular_2(self):
"""
Circular M2M relations with explicit through models should be serializable
This test tests the circularity with explicit natural_key.dependencies
"""
try:
sorted_deps = sort_dependencies([('fixtures_regress', [
M2MComplexCircular2A, M2MComplexCircular2B,
M2MCircular2ThroughAB
])])
except CommandError:
self.fail("Serialization dependency solving algorithm isn't "
"capable of handling circular M2M setups with "
"intermediate models plus natural key dependency hints.")
self.assertEqual(sorted_deps[:2],
[M2MComplexCircular2A, M2MComplexCircular2B])
self.assertEqual(sorted_deps[2:], [M2MCircular2ThroughAB])
def test_dependency_sorting_m2m_complex_circular_2(self):
"""
Circular M2M relations with explicit through models should be serializable
This test tests the circularity with explicit natural_key.dependencies
"""
try:
sorted_deps = sort_dependencies([
('fixtures_regress', [
M2MComplexCircular2A,
M2MComplexCircular2B,
M2MCircular2ThroughAB])
])
except CommandError:
self.fail("Serialization dependency solving algorithm isn't "
"capable of handling circular M2M setups with "
"intermediate models plus natural key dependency hints.")
self.assertEqual(sorted_deps[:2], [M2MComplexCircular2A, M2MComplexCircular2B])
self.assertEqual(sorted_deps[2:], [M2MCircular2ThroughAB])
def handle(self, *app_labels, **options):
self.options = options
self.app_labels = app_labels
self.relateds = []
format = self.options.get('format')
indent = self.options.get('indent')
show_traceback = self.options.get('traceback')
use_natural_keys = self.options.get('use_natural_keys')
self.excluded = self.options.get('excluded').split(',')
# Check that the serialization format exists; this is a shortcut to
# avoid collating all the objects and _then_ failing.
if format not in serializers.get_public_serializer_formats():
raise CommandError("Unknown serialization format: %s" % format)
try:
serializers.get_serializer(format)
except KeyError:
raise CommandError("Unknown serialization format: %s" % format)
# Now collate the objects to be serialized.
self.dump_list = InstancesList()
for qs in self.get_querysets():
for item in qs:
self.extract_fields(item)
app_list = self.dump_list.model_list()
obj_list = []
ordered = sort_dependencies(app_list.items())
explored = []
for item in ordered:
if item not in explored:
app_list = self.dump_list.get_items_by_model_path(item)
obj_list.extend(app_list)
explored.append(item)
try:
return serializers.serialize(format,
obj_list,
indent=indent,
use_natural_keys=use_natural_keys)
except Exception, e:
if show_traceback:
raise
raise CommandError("Unable to serialize database: %s" % e)
def get_objects():
"""
Collate the objects for serialization, taken from dumpdata command
and adjusted to sanitize User password hashes
"""
# Collate the objects to be serialized.
for model in sort_dependencies(app_list.items()):
if not model._meta.proxy and router.allow_migrate(
DEFAULT_DB_ALIAS, model):
objects = model._base_manager
queryset = objects.using(DEFAULT_DB_ALIAS).order_by(
model._meta.pk.name)
for obj in queryset.iterator():
# Sanitize user objects
if model._meta.model_name == 'user':
obj.set_unusable_password()
yield obj
def handle(self, *app_labels, **options):
self.options = options
self.app_labels = app_labels
self.relateds = []
format = self.options.get('format')
indent = self.options.get('indent')
show_traceback = self.options.get('traceback')
use_natural_keys = self.options.get('use_natural_keys')
self.excluded = self.options.get('excluded').split(',')
# Check that the serialization format exists; this is a shortcut to
# avoid collating all the objects and _then_ failing.
if format not in serializers.get_public_serializer_formats():
raise CommandError("Unknown serialization format: %s" % format)
try:
serializers.get_serializer(format)
except KeyError:
raise CommandError("Unknown serialization format: %s" % format)
# Now collate the objects to be serialized.
self.dump_list = InstancesList()
for qs in self.get_querysets():
for item in qs:
self.extract_fields(item)
app_list = self.dump_list.model_list()
obj_list = []
ordered = sort_dependencies(app_list.items())
explored = []
for item in ordered:
if item not in explored:
app_list = self.dump_list.get_items_by_model_path(item)
obj_list.extend(app_list)
explored.append(item)
try:
return serializers.serialize(format, obj_list,
indent=indent,
use_natural_keys=use_natural_keys)
except Exception, e:
if show_traceback:
raise
raise CommandError("Unable to serialize database: %s" % e)
def test_dependency_sorting_m2m_complex_circular_1(self):
"""
Circular M2M relations with explicit through models should be serializable
"""
A, B, C, AtoB, BtoC, CtoA = (M2MComplexCircular1A, M2MComplexCircular1B,
M2MComplexCircular1C, M2MCircular1ThroughAB,
M2MCircular1ThroughBC, M2MCircular1ThroughCA)
try:
sorted_deps = sort_dependencies(
[('fixtures_regress', [A, B, C, AtoB, BtoC, CtoA])]
)
except CommandError:
self.fail("Serialization dependency solving algorithm isn't "
"capable of handling circular M2M setups with "
"intermediate models.")
# The dependency sorting should not result in an error, and the
# through model should have dependencies to the other models and as
# such come last in the list.
self.assertEqual(sorted_deps[:3], [A, B, C])
self.assertEqual(sorted_deps[3:], [AtoB, BtoC, CtoA])
def data_dump(request):
if not request.user.is_authenticated() or not request.user.is_superuser:
response = HttpResponse(json.dumps({"Error": "Not Authorized"}))
response['Content-type'] = 'application/json'
return response
#if wanted all apps, but only want campus
#app_list = SortedDict([(app, None) for app in get_apps()])
app_list = SortedDict([(get_app('campus'), None)])
# Now collate the objects to be serialized.
objects = []
# Needed because sqllite doesn't use
def ordering(self):
if hasattr(self, 'name'):
return str(self.name).lower()
elif hasattr(self, 'id'):
return self.id
else:
return self.pk
for model in sort_dependencies(app_list.items()):
# skip groupedlocation model (not needed since Group uses natural keys)
if model == GroupedLocation:
continue
# - make ordering case insensitive
# - must also make special case for MapObj else the leaf class will be
# serialized, not the actual MapObj itself
if model == MapObj:
objects.extend(sorted(model.objects.mob_filter(), key=ordering))
else:
objects.extend(sorted(model.objects.all(), key=ordering))
try:
data = serializers.serialize('json',
objects,
indent=4,
use_natural_keys=True)
except Exception, e:
data = serializers.serialize('json', "ERORR!")
def _update_models(self):
if (self.process_before() == False):
# 前処理
return
# INSTALLED_APPから全モデルクラスを取得
self._using = DEFAULT_DB_ALIAS # ToDo: defaultだけではなく、マルチDB化
models = sort_dependencies(self._app_list.items())
ofn = "/tmp/django_s3_backup_%s_" % (
datetime.now().strftime("%Y%m%d%H%M%S"))
# print "Writing JSON to %s..." % ofn
print "get all models Done."
# appのmodel毎にjsonダンプ -> S3に保存
stream_fp = None
for model in models:
# dbg start
#if ( model.__name__ != "Customer" ):
# continue
# dbg end
if ((not model._meta.proxy)
and (router.allow_syncdb(self._using, model))):
try:
# 全レコードをファイルofnへ書き出し
fsize = self.create_json_file(ofn, model, self._using)
if (fsize == None):
# レコード存在しない場合
continue
# ファイル内容をS3にアップロード
self.update_S3(model, ofn, fsize)
except:
if (stream_fp):
stream_fp.close()
os.remove(ofn)
raise
if (self.process_after() == False):
# 後処理
return
def test_dependency_sorting_m2m_complex_circular_1(self):
"""
Circular M2M relations with explicit through models should be serializable
"""
A, B, C, AtoB, BtoC, CtoA = (M2MComplexCircular1A,
M2MComplexCircular1B,
M2MComplexCircular1C,
M2MCircular1ThroughAB,
M2MCircular1ThroughBC,
M2MCircular1ThroughCA)
try:
sorted_deps = sort_dependencies([('fixtures_regress',
[A, B, C, AtoB, BtoC, CtoA])])
except CommandError:
self.fail("Serialization dependency solving algorithm isn't "
"capable of handling circular M2M setups with "
"intermediate models.")
# The dependency sorting should not result in an error, and the
# through model should have dependencies to the other models and as
# such come last in the list.
self.assertEqual(sorted_deps[:3], [A, B, C])
self.assertEqual(sorted_deps[3:], [AtoB, BtoC, CtoA])
def data_dump(request):
if not request.user.is_authenticated() or not request.user.is_superuser:
response = HttpResponse(json.dumps({"Error": "Not Authorized"}))
response['Content-type'] = 'application/json'
return response
#if wanted all apps, but only want campus
#app_list = SortedDict([(app, None) for app in get_apps()])
app_list = SortedDict([(get_app('campus'), None)])
# Now collate the objects to be serialized.
objects = []
# Needed because sqllite doesn't use
def ordering(self):
if hasattr(self, 'name'):
return str(self.name).lower()
elif hasattr(self, 'id'):
return self.id
else:
return self.pk
for model in sort_dependencies(app_list.items()):
# skip groupedlocation model (not needed since Group uses natural keys)
if model == GroupedLocation:
continue
# - make ordering case insensitive
# - must also make special case for MapObj else the leaf class will be
# serialized, not the actual MapObj itself
if model == MapObj:
objects.extend( sorted(model.objects.mob_filter(), key=ordering) )
else:
objects.extend( sorted(model.objects.all(), key=ordering) )
try:
data = serializers.serialize('json', objects, indent=4, use_natural_keys=True)
except Exception, e:
data = serializers.serialize('json', "ERORR!")
def dump_data(request, appname):
app_list = SortedDict()
try:
if request.method == 'POST':
for appname in request.POST.getlist('apps'):
app = get_app(appname)
app_list[app] = None
appname = 'choices'
else:
app = get_app(appname)
app_list[app] = None
except ImproperlyConfigured:
if appname == 'all':
for app in get_apps():
app_list[app] = None
if (len(app_list) > 0):
objects = []
for model in sort_dependencies(app_list.items()):
if not model._meta.proxy and router.allow_syncdb(
DEFAULT_DB_ALIAS, model):
objects.extend(
model._default_manager.using(DEFAULT_DB_ALIAS).all())
serializers.get_serializer('json')
json = serializers.serialize('json',
objects,
indent=2,
use_natural_keys=True)
response = HttpResponse(json, mimetype='application/json')
response[
'Content-Disposition'] = 'attachment; filename=%s_%s_fixture.json' % (
date.today().__str__(), appname)
return response
return render_to_response('diagnostic/dumpdata.html',
context_instance=RequestContext(request))
def _update_models(self):
if ( self.process_before() == False ):
# 前処理
return
# INSTALLED_APPから全モデルクラスを取得
self._using = DEFAULT_DB_ALIAS # ToDo: defaultだけではなく、マルチDB化
models = sort_dependencies(self._app_list.items())
ofn = "/tmp/django_s3_backup_%s_" % ( datetime.now().strftime("%Y%m%d%H%M%S") )
# print "Writing JSON to %s..." % ofn
print "get all models Done."
# appのmodel毎にjsonダンプ -> S3に保存
stream_fp = None
for model in models:
# dbg start
#if ( model.__name__ != "Customer" ):
# continue
# dbg end
if ( (not model._meta.proxy) and (router.allow_syncdb(self._using, model)) ):
try:
# 全レコードをファイルofnへ書き出し
fsize = self.create_json_file(ofn, model, self._using)
if ( fsize == None ):
# レコード存在しない場合
continue
# ファイル内容をS3にアップロード
self.update_S3(model, ofn, fsize)
except:
if ( stream_fp ):
stream_fp.close()
os.remove(ofn)
raise
if ( self.process_after() == False ):
# 後処理
return
def test_dependency_sorting_3(self):
sorted_deps = sort_dependencies([("fixtures_regress", [Store, Book, Person])])
self.assertEqual(sorted_deps, [Store, Person, Book])
def handle(self, *args, **options):
""" Dump the objects, Luke. """
def empty_parameters(processor):
""" Remove processors parameters if they come from disk. """
if bool(processor.source_uri):
processor.parameters = None
return processor
limiting_usernames = os.environ.get('SELECTIVE_DUMP_1FLOW_USERS',
u'admin').split(',')
limiting_categories = os.environ.get('SELECTIVE_DUMP_1FLOW_CATEGORIES',
u'1flow-stock').split(',')
fixture_filename = os.path.join(
settings.PROJECT_ROOT, 'core', 'fixtures',
u'core_processors_pack__{0}__{1}.json'.format(
u','.join(limiting_usernames),
u','.join(limiting_categories),
)
)
limiting_query1 = (
Q(slug__in=limiting_categories)
| Q(user__username__in=limiting_usernames)
)
limiting_query2 = (
Q(categories__slug__in=limiting_categories)
| Q(user__username__in=limiting_usernames)
)
# Use OrderedDict because sort_dependencies() doesn't
# work as expected. See original SO answer for details.
app_list = OrderedDict()
total_count = 0
app_list['core.ProcessorCategory'] = \
ProcessorCategory.objects.filter(limiting_query1).order_by('id')
total_count += app_list['core.ProcessorCategory'].count()
app_list['core.Processor'] = [
empty_parameters(p)
for p in Processor.objects.filter(limiting_query2).order_by('id')
]
total_count += len(app_list['core.Processor'])
app_list['core.ProcessChain'] = \
ProcessingChain.objects.filter(limiting_query2).order_by('id')
total_count += app_list['core.ProcessChain'].count()
app_list['core.ChainedItem'] = \
ChainedItem.objects.filter(
chain__in=app_list['core.ProcessChain']).order_by(
'chain', 'position').order_by('id')
total_count += app_list['core.ChainedItem'].count()
# ——————————————————————————————————————————————————————— Serialization
data = serializers.serialize('json',
sort_dependencies(app_list.items()),
indent=2, use_natural_keys=True)
with open(fixture_filename, 'w') as f:
f.write(data)
self.stdout.write(u'Exported %s items in %s.'
% (total_count, fixture_filename))
