def test_dependency_self_referential(self):
with self.assertRaisesMessage(
RuntimeError,
"Can't resolve dependencies for fixtures_regress.Circle3 in "
"serialized app list."
):
serializers.sort_dependencies([('fixtures_regress', [Book, Circle3])])
def test_dependency_sorting_tight_circular_2(self):
with self.assertRaisesMessage(
RuntimeError,
"Can't resolve dependencies for fixtures_regress.Circle1, "
"fixtures_regress.Circle2 in serialized app list."
):
serializers.sort_dependencies([('fixtures_regress', [Circle1, Book, Circle2])])
def test_dependency_sorting_long(self):
with self.assertRaisesMessage(
RuntimeError,
"Can't resolve dependencies for fixtures_regress.Circle1, "
"fixtures_regress.Circle2, fixtures_regress.Circle3 in serialized "
"app list."
):
serializers.sort_dependencies([('fixtures_regress', [Person, Circle2, Circle1, Circle3, Store, Book])])
def test_dependency_sorting_m2m_simple_circular(self):
"""
Resolving circular M2M relations without explicit through models should
fail loudly
"""
with self.assertRaisesMessage(
RuntimeError,
"Can't resolve dependencies for fixtures_regress.M2MSimpleCircularA, "
"fixtures_regress.M2MSimpleCircularB in serialized app list."
):
serializers.sort_dependencies([('fixtures_regress', [M2MSimpleCircularA, M2MSimpleCircularB])])
def get_objects(count_only=False):
"""
Collate the objects to be serialized. If count_only is True, just
count the number of objects to be serialized.
"""
models = serializers.sort_dependencies(app_list.items())
for model in models:
if model in excluded_models:
continue
if model._meta.proxy and model._meta.proxy_for_model not in models:
warnings.warn(
"%s is a proxy model and won't be serialized." % model._meta.label,
category=ProxyModelWarning,
)
if not model._meta.proxy and router.allow_migrate_model(using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = model._default_manager
queryset = objects.using(using).order_by(model._meta.pk.name)
if primary_keys:
queryset = queryset.filter(pk__in=primary_keys)
if count_only:
yield queryset.order_by().count()
else:
for obj in queryset.iterator():
yield obj
def _dump():
s_time = time.time()
from django.apps import apps
from django.db.models import get_model
excluded_models = set()
for exclude in EXCLUDED_MODELS:
app_label, model_name = exclude.split('.', 1)
model_obj = get_model(app_label, model_name)
excluded_models.add(model_obj)
app_list = [(c, None) for c in apps.get_app_configs() if c.label in
set(APPS_TO_HANDLE)]
objects = []
for model in serializers.sort_dependencies(app_list):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_migrate(DEFAULT_DB_ALIAS, model):
objects.extend(model._default_manager.using(DEFAULT_DB_ALIAS).all())
f_name = DUMP_NAME.format(datetime.now())
with closing(StringIO()) as compressed_data:
with gzip.GzipFile(filename=f_name, mode='wb',
fileobj=compressed_data) as compressor:
compressor.write(serializers.serialize('json', objects, indent=2,
use_natural_foreign_keys=True))
compressed_data.seek(0)
default_storage.save(DUMP_PATH + f_name + '.gz', compressed_data)
return '%d objects exported to %s in %d seconds'%(len(objects), f_name,
time.time() - s_time)
def get_objects():
for model in serializers.sort_dependencies(app_list):
if (not model._meta.proxy and model._meta.managed and
router.allow_migrate(self.connection.alias, model)):
queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
for obj in queryset.iterator():
yield obj
def test_dependency_sorting_5(self):
sorted_deps = serializers.sort_dependencies(
[('fixtures_regress', [Person, Book, Store])]
)
self.assertEqual(
sorted_deps,
[Store, Person, Book]
)
def test_dependency_sorting_dangling(self):
sorted_deps = serializers.sort_dependencies(
[('fixtures_regress', [Person, Circle1, Store, Book])]
)
self.assertEqual(
sorted_deps,
[Circle1, Store, Person, Book]
)
def test_dependency_sorting_normal(self):
sorted_deps = serializers.sort_dependencies(
[('fixtures_regress', [Person, ExternalDependency, Book])]
)
self.assertEqual(
sorted_deps,
[Person, Book, ExternalDependency]
)
def test_dependency_sorting_m2m_complex_circular_2(self):
"""
Circular M2M relations with explicit through models should be serializable
This test tests the circularity with explicit natural_key.dependencies
"""
sorted_deps = serializers.sort_dependencies([
('fixtures_regress', [M2MComplexCircular2A, M2MComplexCircular2B, M2MCircular2ThroughAB])
])
self.assertEqual(sorted_deps[:2], [M2MComplexCircular2A, M2MComplexCircular2B])
self.assertEqual(sorted_deps[2:], [M2MCircular2ThroughAB])
def test_dependency_sorting_m2m_complex(self):
"""
M2M relations with explicit through models should NOT count as
dependencies. The through model itself will have dependencies, though.
"""
sorted_deps = serializers.sort_dependencies(
[('fixtures_regress', [M2MComplexA, M2MComplexB, M2MThroughAB])]
)
# Order between M2MComplexA and M2MComplexB doesn't matter. The through
# model has dependencies to them though, so it should come last.
self.assertEqual(sorted_deps[-1], M2MThroughAB)
def test_dependency_sorting(self):
"""
It doesn't matter what order you mention the models, Store *must* be
serialized before then Person, and both must be serialized before Book.
"""
sorted_deps = serializers.sort_dependencies(
[('fixtures_regress', [Book, Person, Store])]
)
self.assertEqual(
sorted_deps,
[Store, Person, Book]
)
def test_dependency_sorting_m2m_simple(self):
"""
M2M relations without explicit through models SHOULD count as dependencies
Regression test for bugs that could be caused by flawed fixes to
#14226, namely if M2M checks are removed from sort_dependencies
altogether.
"""
sorted_deps = serializers.sort_dependencies(
[('fixtures_regress', [M2MSimpleA, M2MSimpleB])]
)
self.assertEqual(sorted_deps, [M2MSimpleB, M2MSimpleA])
def getCoreModelClasses(module):
from chembl_core_db.db.models.abstractModel import ChemblCoreAbstractModel
from django.core.serializers import sort_dependencies
if django_version < "1.7.0":
from django.db.models import get_app
from django.utils.datastructures import SortedDict
app_list = SortedDict((app, None) for app in [get_app('chembl_core_model')])
else:
from django.apps import apps
from collections import OrderedDict
app_list = OrderedDict((app, None) for app in [apps.get_app_config('chembl_core_model')])
return [model for model in sort_dependencies(app_list.items()) if
model.__name__ not in EXCLUDED_MODELS and issubclass(model,
ChemblCoreAbstractModel) and not model._meta.abstract]
def test_dependency_sorting_m2m_complex_circular_1(self):
"""
Circular M2M relations with explicit through models should be serializable
"""
A, B, C, AtoB, BtoC, CtoA = (M2MComplexCircular1A, M2MComplexCircular1B,
M2MComplexCircular1C, M2MCircular1ThroughAB,
M2MCircular1ThroughBC, M2MCircular1ThroughCA)
sorted_deps = serializers.sort_dependencies(
[('fixtures_regress', [A, B, C, AtoB, BtoC, CtoA])]
)
# The dependency sorting should not result in an error, and the
# through model should have dependencies to the other models and as
# such come last in the list.
self.assertEqual(sorted_deps[:3], [A, B, C])
self.assertEqual(sorted_deps[3:], [AtoB, BtoC, CtoA])
def get_objects():
# Collate the objects to be serialized.
for model in serializers.sort_dependencies(app_list.items()):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_migrate_model(using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = model._default_manager
queryset = objects.using(using).order_by(model._meta.pk.name)
if primary_keys:
queryset = queryset.filter(pk__in=primary_keys)
for obj in queryset.iterator():
yield obj
def handle(self, *args, **options):
from django.apps import apps
# TODO : Check export mode
db.reset_queries()
sourceDatabase = options.get('sourceDatabase')
dataLimit = options.get('dataLimit')
app = apps.get_app(options.get('app'))
con = connections[sourceDatabase]
if con.vendor != 'oracle':
print "Source database has to be oracle."
return
user = settings.DATABASES[sourceDatabase]['USER']
passwd = settings.DATABASES[sourceDatabase]['PASSWORD']
host = settings.DATABASES[sourceDatabase]['HOST']
port = settings.DATABASES[sourceDatabase]['PORT']
name = settings.DATABASES[sourceDatabase]['NAME']
app_list = OrderedDict((app, None) for app in [app])
tables = []
sorted = sort_dependencies(app_list.items())
lastObjectName = sorted[-1].__name__
filename = lastObjectName + ".postgresql_psycopg2.sql"
chemblSQLPath = os.path.join(os.path.dirname(app.__file__),'sql', filename)
location = chemblSQLPath
oracleHome = os.environ['ORACLE_HOME']
if options.get('dumpfile'):
if not options.get('dumpfile').endswith('.sql'):
location = os.path.join(options.get('dumpfile'), filename)
else:
location = options.get('dumpfile')
for model in reversed(sorted):
if not model._meta.managed:
continue
tables.append(model._meta.db_table)
print self.confTemplate % (oracleHome, host, name, port, user, passwd, user, " ".join(tables), dataLimit, location)
if location != chemblSQLPath:
print "different! location = " + location + ", chemblSQLPath = " + chemblSQLPath
f = open(location, 'w')
f.close()
os.symlink(location, chemblSQLPath)
def get_querysets_to_dump(domain, excludes):
"""
:param domain: domain name to filter with
:param app_list: List of (app_config, model_class) tuples to dump
:param excluded_models: List of model_class classes to exclude
:return: generator yielding query sets
"""
excluded_apps, excluded_models = get_excluded_apps_and_models(excludes)
app_config_models = _get_app_list(excluded_apps)
# Collate the objects to be serialized.
for model_class in serializers.sort_dependencies(app_config_models.items()):
if model_class in excluded_models:
continue
for model_class, queryset in get_all_model_querysets_for_domain(model_class, domain):
yield model_class, queryset
def test_dependency_sorting_m2m_complex_circular_2(self):
"""
Circular M2M relations with explicit through models should be serializable
This test tests the circularity with explicit natural_key.dependencies
"""
try:
sorted_deps = serializers.sort_dependencies([
('fixtures_regress', [
M2MComplexCircular2A,
M2MComplexCircular2B,
M2MCircular2ThroughAB])
])
except CommandError:
self.fail("Serialization dependency solving algorithm isn't "
"capable of handling circular M2M setups with "
"intermediate models plus natural key dependency hints.")
self.assertEqual(sorted_deps[:2], [M2MComplexCircular2A, M2MComplexCircular2B])
self.assertEqual(sorted_deps[2:], [M2MCircular2ThroughAB])
def test_dependency_sorting_m2m_complex_circular_1(self):
"""
Circular M2M relations with explicit through models should be serializable
"""
A, B, C, AtoB, BtoC, CtoA = (M2MComplexCircular1A, M2MComplexCircular1B,
M2MComplexCircular1C, M2MCircular1ThroughAB,
M2MCircular1ThroughBC, M2MCircular1ThroughCA)
try:
sorted_deps = serializers.sort_dependencies(
[('fixtures_regress', [A, B, C, AtoB, BtoC, CtoA])]
)
except CommandError:
self.fail("Serialization dependency solving algorithm isn't "
"capable of handling circular M2M setups with "
"intermediate models.")
# The dependency sorting should not result in an error, and the
# through model should have dependencies to the other models and as
# such come last in the list.
self.assertEqual(sorted_deps[:3], [A, B, C])
self.assertEqual(sorted_deps[3:], [AtoB, BtoC, CtoA])
def test_dependency_sorting_3(self):
sorted_deps = serializers.sort_dependencies(
[("fixtures_regress", [Store, Book, Person])]
)
self.assertEqual(sorted_deps, [Store, Person, Book])
def test_dependency_sorting_5(self):
sorted_deps = serializers.sort_dependencies([('fixtures_regress',
[Person, Book, Store])])
self.assertEqual(sorted_deps, [Store, Person, Book])
def test_dependency_sorting_dangling(self):
sorted_deps = serializers.sort_dependencies([
('fixtures_regress', [Person, Circle1, Store, Book])
])
self.assertEqual(sorted_deps, [Circle1, Store, Person, Book])
def test_dependency_sorting_normal(self):
sorted_deps = serializers.sort_dependencies([
('fixtures_regress', [Person, ExternalDependency, Book])
])
self.assertEqual(sorted_deps, [Person, Book, ExternalDependency])
def get_objects():
for model in serializers.sort_dependencies(app_list):
if (model._meta.can_migrate(self.connection) and
router.allow_migrate_model(self.connection.alias, model)):
queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
yield from queryset.iterator()
import sys
from collections import OrderedDict
def get_objects():
for model in serializers.sort_dependencies(app_list):
if (model._meta.can_migrate(self.connection) and
router.allow_migrate_model(self.connection.alias, model)):
queryset = model._default_manager.using(self.connection.alias).order_by(model._meta.pk.name)
yield from queryset.iterator()
