def show_db_overview(self):
"""Return a reStructredText-formatted "database overview" report.
Used by test cases in tested documents.
"""
from lino.core.utils import (full_model_name, sorted_models_list)
models_list = sorted_models_list()
apps = [p.app_label for p in settings.SITE.installed_plugins]
s = "%d apps: %s." % (len(apps), ", ".join(apps))
s += "\n%d models:\n" % len(models_list)
i = 0
headers = [
#~ "No.",
"Name",
"Default table",
#~ "M",
"#fields",
"#rows",
#~ ,"first","last"
]
rows = []
for model in models_list:
if True: # model._meta.managed:
i += 1
cells = []
#~ cells.append(str(i))
cells.append(full_model_name(model))
cells.append(model.get_default_table())
#~ cells.append(str(model))
#~ if model._meta.managed:
#~ cells.append('X')
#~ else:
#~ cells.append('')
cells.append(str(len(model._meta.concrete_fields)))
qs = model.objects.all()
n = qs.count()
cells.append(str(n))
#~ if n:
#~ cells.append(obj2str(qs[0]))
#~ cells.append(obj2str(qs[n-1]))
#~ else:
#~ cells.append('')
#~ cells.append('')
rows.append(cells)
s += rstgen.table(headers, rows)
return s
def show_db_overview(self):
"""Return a reStructredText-formatted "database overview" report.
Used by test cases in tested documents.
"""
from lino.core.utils import (full_model_name, sorted_models_list)
models_list = sorted_models_list()
apps = [p.app_label for p in settings.SITE.installed_plugins]
s = "%d apps: %s." % (len(apps), ", ".join(apps))
s += "\n%d models:\n" % len(models_list)
i = 0
headers = [
#~ "No.",
"Name",
"Default table",
#~ "M",
"#fields",
"#rows",
#~ ,"first","last"
]
rows = []
for model in models_list:
if True: # model._meta.managed:
i += 1
cells = []
#~ cells.append(str(i))
cells.append(full_model_name(model))
cells.append(model.get_default_table())
#~ cells.append(str(model))
#~ if model._meta.managed:
#~ cells.append('X')
#~ else:
#~ cells.append('')
cells.append(str(len(model._meta.concrete_fields)))
qs = model.objects.all()
n = qs.count()
cells.append(str(n))
#~ if n:
#~ cells.append(obj2str(qs[0]))
#~ cells.append(obj2str(qs[n-1]))
#~ else:
#~ cells.append('')
#~ cells.append('')
rows.append(cells)
s += rstgen.table(headers, rows)
return s
def show_fields_by_type(fldtype):
"""Print a list of all fields (in all models) that have the specified type.
"""
from lino.core.utils import (sorted_models_list)
items = []
for model in sorted_models_list():
flds = []
for f in model._meta.fields:
if isinstance(f, fldtype):
name = f.name
verbose_name = force_text(f.verbose_name).strip()
txt = "{verbose_name} ({name})".format(**locals())
flds.append(txt)
if len(flds):
txt = "{model} : {fields}".format(model=full_model_name(model),
fields=", ".join(flds))
items.append(txt)
print(rstgen.ul(items))
def write_files(self):
puts("Writing {0}...".format(self.main_file))
self.stream = open(self.main_file, "wt")
current_version = settings.SITE.version
self.stream.write(
"""\
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
"""
)
self.stream.write(
'''\
"""
This is a Python dump created using %s.
'''
% settings.SITE.using_text()
)
# ~ self.stream.write(settings.SITE.welcome_text())
self.stream.write(
'''
"""
from __future__ import unicode_literals
import logging
logger = logging.getLogger('%s')
import os
'''
% __name__
)
if False:
self.stream.write(
"""
os.environ['DJANGO_SETTINGS_MODULE'] = '%s'
"""
% settings.SETTINGS_MODULE
)
self.stream.write("SOURCE_VERSION = %r\n" % str(current_version))
self.stream.write(
"""
from decimal import Decimal
from datetime import datetime as dt
from datetime import time, date
from django.conf import settings
# from django.contrib.contenttypes.models import ContentType
from lino.utils.dpy import create_mti_child
from lino.utils.dpy import DpyLoader
from lino.core.utils import resolve_model
def new_content_type_id(m):
if m is None: return m
ct = settings.SITE.modules.contenttypes.ContentType.objects.get_for_model(m)
if ct is None: return None
return ct.pk
"""
)
s = ",".join(["%s=values[%d]" % (lng.name, lng.index) for lng in settings.SITE.languages])
self.stream.write(
'''
def bv2kw(fieldname, values):
"""
Needed if `Site.languages` changed between dumpdata and loaddata
"""
return settings.SITE.babelkw(fieldname, %s)
'''
% s
)
self.models = sorted_models_list()
if settings.SITE.is_installed("contenttypes"):
from django.contrib.contenttypes.models import ContentType
self.models = [m for m in self.models if not issubclass(m, ContentType)]
if settings.SITE.is_installed("sessions"):
from django.contrib.sessions.models import Session
self.models = [m for m in self.models if not issubclass(m, Session)]
for model in self.models:
self.stream.write('%s = resolve_model("%s")\n' % (full_model_name(model, "_"), full_model_name(model)))
self.stream.write("\n")
self.models = self.sort_models(self.models)
self.stream.write("\n")
for model in self.models:
if AFTER17:
fields = [f for f in model._meta.get_fields() if f.concrete and f.model is model]
else:
fields = [f for f, m in model._meta.get_fields_with_model() if m is None]
for f in fields:
if getattr(f, "auto_now_add", False):
raise Exception(
"%s.%s.auto_now_add is True : values will be lost!" % (full_model_name(model), f.name)
)
# ~ fields = model._meta.local_fields
# ~ fields = [f for f in model._meta.fields if f.serialize]
# ~ fields = [f for f in model._meta.local_fields if f.serialize]
self.stream.write(
"def create_%s(%s):\n"
% (
model._meta.db_table,
", ".join([f.attname for f in fields if not getattr(f, "_lino_babel_field", False)]),
)
)
for f in fields:
if isinstance(f, models.DecimalField):
self.stream.write(" if %s is not None: %s = Decimal(%s)\n" % (f.attname, f.attname, f.attname))
elif isinstance(f, ChoiceListField):
lstname = "settings.SITE.modules.{0}.{1}".format(f.choicelist.app_label, f.choicelist.__name__)
ln = " if {0}: {0} = {1}.get_by_value({0})\n"
self.stream.write(ln.format(f.attname, lstname))
elif is_pointer_to_contenttype(f):
self.stream.write(" %s = new_content_type_id(%s)\n" % (f.attname, f.attname))
if model._meta.parents:
if len(model._meta.parents) != 1:
msg = "%s : model._meta.parents is %r" % (model, model._meta.parents)
raise Exception(msg)
pm, pf = list(model._meta.parents.items())[0]
child_fields = [f for f in fields if f != pf]
if child_fields:
attrs = "," + ",".join(["%s=%s" % (f.attname, f.attname) for f in child_fields])
else:
attrs = ""
self.stream.write(
" return create_mti_child(%s, %s, %s%s)\n"
% (full_model_name(pm, "_"), pf.attname, full_model_name(model, "_"), attrs)
)
else:
self.stream.write(" kw = dict()\n")
for f in fields:
if getattr(f, "_lino_babel_field", False):
continue
elif isinstance(f, (BabelCharField, BabelTextField)):
self.stream.write(
" if %s is not None: kw.update(bv2kw(%r,%s))\n" % (f.attname, f.attname, f.attname)
)
else:
self.stream.write(" kw.update(%s=%s)\n" % (f.attname, f.attname))
self.stream.write(" return %s(**kw)\n\n" % full_model_name(model, "_"))
self.stream.write("\n")
# ~ used_models = set()
self.stream.write(
"""
def main():
loader = DpyLoader(globals())
from django.core.management import call_command
# call_command('initdb', interactive=False)
call_command('initdb')
os.chdir(os.path.dirname(__file__))
loader.initialize()
"""
)
for model in progress.bar(self.models):
filename = "%s.py" % model._meta.db_table
filename = os.path.join(self.output_dir, filename)
# puts("Writing {0}...".format(filename))
stream = file(filename, "wt")
stream.write("# -*- coding: UTF-8 -*-\n")
qs = model.objects.all()
try:
stream.write('logger.info("Loading %d objects to table %s...")\n' % (qs.count(), model._meta.db_table))
if AFTER17:
fields = [f for f in model._meta.get_fields() if f.concrete and f.model is model]
else:
fields = [f for f, m in model._meta.get_fields_with_model() if m is None]
fields = [f for f in fields if not getattr(f, "_lino_babel_field", False)]
stream.write("# fields: %s\n" % ", ".join([f.name for f in fields]))
for obj in qs:
self.count_objects += 1
# ~ used_models.add(model)
stream.write(
"loader.save(create_%s(%s))\n"
% (obj._meta.db_table, ",".join([self.value2string(obj, f) for f in fields]))
)
stream.write("\n")
stream.write("loader.flush_deferred_objects()\n")
except ProgrammingError as e:
if not self.options["tolerate"]:
raise
stream.write("\n")
logger.warning("Tolerating database error %s in %s", e, model._meta.db_table)
msg = ("The data of this table has not been dumped" "because an error {0} occured.").format(e)
stream.write('raise Exception("{0}")\n'.format(msg))
stream.close()
# ~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % )
self.stream.write(' execfile("%s.py")\n' % model._meta.db_table)
self.stream.write(" loader.finalize()\n")
# self.stream.write(
# ' logger.info("Loaded %d objects",loader.count_objects)\n')
self.stream.write(
"""
if __name__ == '__main__':
main()
"""
)
# ~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n')
self.stream.close()
def handle(self, *args, **options):
if args:
raise CommandError("This command doesn't accept any arguments.")
self.options = options
#~ settings.SITE.startup()
state = dict()
state.update(timestamp=datetime.datetime.now())
state.update(lino_version=lino.__version__)
states_file = os.path.join(settings.SITE.project_dir, 'states.pck')
if os.path.exists(states_file):
fd = open(states_file)
states_list = pickle.load(fd)
fd.close()
logger.info("Loaded %d states from %s",
len(states_list), states_file)
else:
states_list = []
models_list = sorted_models_list()
apps = [p.app_label for p in settings.SITE.installed_plugins]
state.update(applications=" ".join(apps))
for model in models_list:
if model._meta.managed:
model_state = dict()
#~ cells.append(str(i))
#~ cells.append(full_model_name(model))
#~ cells.append(str(model))
#~ if model._meta.managed:
#~ cells.append('X')
#~ else:
#~ cells.append('')
model_state.update(fields=[f.name for f in model._meta.fields])
#~ qs = model.objects.all()
qs = model.objects.order_by('pk')
n = qs.count()
model_state.update(rows=n)
connection = connections[DEFAULT_DB_ALIAS]
#~ if isinstance(connection,sqlite):
#~ cells.append("-")
if mysql and isinstance(connection, mysql):
cursor = connection.cursor()
dbname = connection.settings_dict['NAME']
sql = """\
SELECT (data_length+index_length) tablesize
FROM information_schema.tables
WHERE table_schema='%s' and table_name='%s';
""" % (dbname, model._meta.db_table)
#~ print sql
cursor.execute(sql)
row = cursor.fetchone()
if row is not None:
model_state.update(bytes=row[0])
else:
pass
state[full_model_name(model)] = model_state
if len(states_list):
msg = compare(state, states_list[-1])
if msg:
logger.info(msg)
#~ sendmail_admins()
states_list.append(state)
#~ print state
if self.options['write']:
f = open(states_file, 'w')
pickle.dump(states_list, f)
logger.info("Saved %d states to %s", len(states_list), states_file)
def serialize(self, queryset, **options):
self.options = options
self.stream = options.get("stream", StringIO())
self.selected_fields = options.get("fields")
self.use_natural_keys = options.get("use_natural_keys", False)
if self.write_preamble:
current_version = settings.SITE.version
if '+' in current_version:
logger.warning("Dumpdata from intermediate version %s" %
current_version)
self.stream.write('# -*- coding: UTF-8 -*-\n')
self.stream.write('''\
"""
This is a `Python dump created using %s.
''' % settings.SITE.using_text())
#~ self.stream.write(settings.SITE.welcome_text())
self.stream.write('''
"""
from __future__ import unicode_literals
''')
self.stream.write('SOURCE_VERSION = %r\n' % str(current_version))
self.stream.write('from decimal import Decimal\n')
self.stream.write('from datetime import datetime as dt\n')
self.stream.write('from datetime import time,date\n')
self.stream.write('from lino.utils.dpy import create_mti_child\n')
self.stream.write('from lino.utils.dbutils import resolve_model\n')
self.stream.write(
'from lino.modlib.gfks.models import ContentType\n')
self.stream.write('from django.conf import settings\n')
self.stream.write('''
def new_content_type_id(m):
if m is None: return m
# if not fmn: return None
# m = resolve_model(fmn)
ct = ContentType.objects.get_for_model(m)
if ct is None: return None
return ct.pk
''')
#~ s = ','.join([
#~ '%s=values[%d]' % (k,i)
#~ for i,k in enumerate(settings.SITE.AVAILABLE_LANGUAGES)])
s = ','.join([
'%s=values[%d]' % (lng.name, lng.index)
for lng in settings.SITE.languages
])
self.stream.write('''
def bv2kw(fieldname,values):
"""
Needed if `Site.languages` changed between dumpdata and loaddata
"""
return settings.SITE.babelkw(fieldname,%s)
''' % s)
#~ model = queryset.model
if self.models is None:
self.models = sorted_models_list() # models.get_models()
if self.write_preamble:
for model in self.models:
self.stream.write(
'%s = resolve_model("%s")\n' %
(full_model_name(model, '_'), full_model_name(model)))
self.stream.write('\n')
for model in self.models:
fields = [
f for f, m in model._meta.get_fields_with_model() if m is None
]
for f in fields:
if getattr(f, 'auto_now_add', False):
raise Exception(
"%s.%s.auto_now_add is True : values will be lost!" %
(full_model_name(model), f.name))
field_names = [
f.attname for f in fields
if not getattr(f, '_lino_babel_field', False)
]
self.stream.write('def create_%s(%s):\n' %
(model._meta.db_table, ', '.join(field_names)))
if model._meta.parents:
if len(model._meta.parents) != 1:
msg = "%s : model._meta.parents is %r" % (
model, model._meta.parents)
raise Exception(msg)
pm, pf = model._meta.parents.items()[0]
child_fields = [f for f in fields if f != pf]
if child_fields:
attrs = ',' + ','.join([
'%s=%s' % (f.attname, f.attname) for f in child_fields
])
else:
attrs = ''
tpl = ' return create_mti_child(%s, %s, %s%s)\n'
self.stream.write(tpl % (full_model_name(
pm, '_'), pf.attname, full_model_name(model, '_'), attrs))
else:
self.stream.write(" kw = dict()\n")
for f in fields:
if getattr(f, '_lino_babel_field', False):
continue
elif isinstance(f, (BabelCharField, BabelTextField)):
tpl = ' if %s is not None:'
tpl += ' kw.update(bv2kw(%r, %s))\n'
self.stream.write(tpl %
(f.attname, f.attname, f.attname))
else:
if isinstance(f, models.DecimalField):
self.stream.write(
' if %s is not None: %s = Decimal(%s)\n' %
(f.attname, f.attname, f.attname))
elif isinstance(
f,
models.ForeignKey) and f.rel.to is ContentType:
#~ self.stream.write(
#~ ' %s = ContentType.objects.get_for_model(%s).pk\n' % (
#~ f.attname,f.attname))
self.stream.write(
' %s = new_content_type_id(%s)\n' %
(f.attname, f.attname))
self.stream.write(' kw.update(%s=%s)\n' %
(f.attname, f.attname))
self.stream.write(' return %s(**kw)\n\n' %
full_model_name(model, '_'))
#~ self.start_serialization()
self.stream.write('\n')
model = None
all_models = []
for obj in queryset:
if isinstance(obj, ContentType):
continue
if isinstance(obj, Session):
continue
#~ if isinstance(obj,Permission): continue
if obj.__class__ != model:
model = obj.__class__
if model in all_models:
raise Exception("%s instances weren't grouped!" % model)
all_models.append(model)
self.stream.write('\ndef %s_objects():\n' %
model._meta.db_table)
fields = [
f for f, m in model._meta.get_fields_with_model() if m is None
]
fields = [
f for f in fields if not getattr(f, '_lino_babel_field', False)
]
self.stream.write(
' yield create_%s(%s)\n' % (obj._meta.db_table, ','.join(
[self.value2string(obj, f) for f in fields])))
self.stream.write('\n\ndef objects():\n')
all_models = self.sort_models(all_models)
for model in all_models:
#~ self.stream.write(' for o in %s_objects(): yield o\n' % model._meta.db_table)
self.stream.write(' yield %s_objects()\n' %
model._meta.db_table)
def write_files(self):
puts("Writing {0}...".format(self.main_file))
self.stream = open(self.main_file, 'wt')
current_version = settings.SITE.version
self.stream.write('''\
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# This is a Python dump created using dump2py.
# DJANGO_SETTINGS_MODULE was %r, TIME_ZONE was %r.
''' % (settings.SETTINGS_MODULE, settings.TIME_ZONE))
self.stream.write('''
from __future__ import unicode_literals
import logging
logger = logging.getLogger('%s')
import os
''' % __name__)
self.stream.write('SOURCE_VERSION = %r\n' % str(current_version))
self.stream.write('''
from decimal import Decimal
from datetime import datetime
from datetime import time, date
from django.conf import settings
from django.utils.timezone import make_aware
# from django.contrib.contenttypes.models import ContentType
from lino.utils.dpy import create_mti_child
from lino.utils.dpy import DpyLoader
from lino.core.utils import resolve_model
if settings.USE_TZ:
def dt(*args):
return make_aware(datetime(*args))
else:
def dt(*args):
return datetime(*args)
def new_content_type_id(m):
if m is None: return m
ct = settings.SITE.models.contenttypes.ContentType.objects.get_for_model(m)
if ct is None: return None
return ct.pk
''')
s = ','.join([
'%s=values[%d]' % (lng.name, lng.index)
for lng in settings.SITE.languages
])
self.stream.write('''
def bv2kw(fieldname, values):
"""
Needed if `Site.languages` changed between dumpdata and loaddata
"""
return settings.SITE.babelkw(fieldname, %s)
''' % s)
self.models = sorted_models_list()
if settings.SITE.is_installed('contenttypes'):
from django.contrib.contenttypes.models import ContentType
self.models = [
m for m in self.models if not issubclass(m, ContentType)
]
if settings.SITE.is_installed('sessions'):
from django.contrib.sessions.models import Session
self.models = [
m for m in self.models if not issubclass(m, Session)
]
for model in self.models:
self.stream.write(
'%s = resolve_model("%s")\n' %
(full_model_name(model, '_'), full_model_name(model)))
self.stream.write('\n')
self.models = self.sort_models(self.models)
self.stream.write('\n')
for model in self.models:
fields = [
f for f in model._meta.get_fields()
if f.concrete and f.model is model
]
for f in fields:
if getattr(f, 'auto_now_add', False):
raise Exception(
"%s.%s.auto_now_add is True : values will be lost!" %
(full_model_name(model), f.name))
#~ fields = model._meta.local_fields
#~ fields = [f for f in model._meta.fields if f.serialize]
#~ fields = [f for f in model._meta.local_fields if f.serialize]
self.stream.write('def create_%s(%s):\n' %
(model._meta.db_table, ', '.join([
f.attname for f in fields
if not getattr(f, '_lino_babel_field', False)
])))
for f in fields:
if isinstance(f, models.DecimalField):
self.stream.write(
' if %s is not None: %s = Decimal(%s)\n' %
(f.attname, f.attname, f.attname))
elif isinstance(f, ChoiceListField):
lstname = 'settings.SITE.modules.{0}.{1}'.format(
f.choicelist.app_label, f.choicelist.__name__)
ln = ' if {0}: {0} = {1}.get_by_value({0})\n'
self.stream.write(ln.format(f.attname, lstname))
elif is_pointer_to_contenttype(f):
self.stream.write(' %s = new_content_type_id(%s)\n' %
(f.attname, f.attname))
if model._meta.parents:
if len(model._meta.parents) != 1:
msg = "%s : model._meta.parents is %r" % (
model, model._meta.parents)
raise Exception(msg)
pm, pf = list(model._meta.parents.items())[0]
child_fields = [f for f in fields if f != pf]
if child_fields:
attrs = ',' + ','.join([
'%s=%s' % (f.attname, f.attname) for f in child_fields
])
else:
attrs = ''
self.stream.write(
' return create_mti_child(%s, %s, %s%s)\n' %
(full_model_name(pm, '_'), pf.attname,
full_model_name(model, '_'), attrs))
else:
self.stream.write(" kw = dict()\n")
for f in fields:
if getattr(f, '_lino_babel_field', False):
continue
elif isinstance(f, (BabelCharField, BabelTextField)):
self.stream.write(
' if %s is not None: kw.update(bv2kw(%r,%s))\n'
% (f.attname, f.attname, f.attname))
else:
self.stream.write(' kw.update(%s=%s)\n' %
(f.attname, f.attname))
self.stream.write(' return %s(**kw)\n\n' %
full_model_name(model, '_'))
self.stream.write('\n')
#~ used_models = set()
self.stream.write("""
def main(args):
loader = DpyLoader(globals())
from django.core.management import call_command
call_command('initdb', interactive=args.interactive)
os.chdir(os.path.dirname(__file__))
loader.initialize()
""")
for model in progress.bar(self.models):
filename = '%s.py' % model._meta.db_table
filename = os.path.join(self.output_dir, filename)
# puts("Writing {0}...".format(filename))
# stream = file(filename, 'wt')
stream = open(filename, 'wt')
stream.write('# -*- coding: UTF-8 -*-\n')
qs = model.objects.all()
try:
stream.write(
'logger.info("Loading %d objects to table %s...")\n' %
(qs.count(), model._meta.db_table))
fields = [
f for f in model._meta.get_fields()
if f.concrete and f.model is model
]
fields = [
f for f in fields
if not getattr(f, '_lino_babel_field', False)
]
stream.write("# fields: %s\n" %
', '.join([f.name for f in fields]))
for obj in qs:
self.count_objects += 1
#~ used_models.add(model)
stream.write(
'loader.save(create_%s(%s))\n' %
(obj._meta.db_table, ','.join(
[self.value2string(obj, f) for f in fields])))
stream.write('\n')
stream.write('loader.flush_deferred_objects()\n')
except DatabaseError as e:
self.database_errors += 1
if not self.options['tolerate']:
raise
stream.write('\n')
logger.warning("Tolerating database error %s in %s", e,
model._meta.db_table)
msg = ("The data of this table has not been dumped"
"because an error {0} occured.").format(e)
stream.write('raise Exception("{0}")\n'.format(msg))
stream.close()
#~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % )
self.stream.write(' execfile("%s.py")\n' % model._meta.db_table)
self.stream.write(' loader.finalize()\n')
# self.stream.write(
# ' logger.info("Loaded %d objects",loader.count_objects)\n')
self.stream.write("""
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='Restore the data.')
parser.add_argument('--noinput', dest='interactive',
action='store_false', default=True,
help="Don't ask for confirmation before flushing the database.")
args = parser.parse_args()
main(args)
""")
#~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n')
self.stream.close()
def write_files(self):
puts("Writing {0}...".format(self.main_file))
self.stream = open(self.main_file, 'wt')
current_version = settings.SITE.version
self.stream.write("""\
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
""")
self.stream.write('''\
"""
This is a Python dump created using %s.
''' % settings.SITE.using_text())
#~ self.stream.write(settings.SITE.welcome_text())
self.stream.write('''
"""
from __future__ import unicode_literals
import logging
logger = logging.getLogger('%s')
import os
''' % __name__)
if False:
self.stream.write("""
os.environ['DJANGO_SETTINGS_MODULE'] = '%s'
""" % settings.SETTINGS_MODULE)
self.stream.write('SOURCE_VERSION = %r\n' % str(current_version))
self.stream.write('''
from decimal import Decimal
from datetime import datetime as dt
from datetime import time,date
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from lino.utils.dpy import create_mti_child
from lino.utils.dpy import DpyLoader
from lino.core.utils import resolve_model
def new_content_type_id(m):
if m is None: return m
ct = ContentType.objects.get_for_model(m)
if ct is None: return None
return ct.pk
''')
s = ','.join([
'%s=values[%d]' % (lng.name, lng.index)
for lng in settings.SITE.languages
])
self.stream.write('''
def bv2kw(fieldname, values):
"""
Needed if `Site.languages` changed between dumpdata and loaddata
"""
return settings.SITE.babelkw(fieldname, %s)
''' % s)
self.models = [
m for m in sorted_models_list()
if not issubclass(m, (ContentType, Session))
]
for model in self.models:
self.stream.write(
'%s = resolve_model("%s")\n' %
(full_model_name(model, '_'), full_model_name(model)))
self.stream.write('\n')
self.models = self.sort_models(self.models)
self.stream.write('\n')
for model in self.models:
fields = [
f for f, m in model._meta.get_fields_with_model() if m is None
]
if AFTER17:
fields = [f for f in fields if f.concrete]
for f in fields:
if getattr(f, 'auto_now_add', False):
raise Exception(
"%s.%s.auto_now_add is True : values will be lost!" %
(full_model_name(model), f.name))
#~ fields = model._meta.local_fields
#~ fields = [f for f in model._meta.fields if f.serialize]
#~ fields = [f for f in model._meta.local_fields if f.serialize]
self.stream.write('def create_%s(%s):\n' %
(model._meta.db_table, ', '.join([
f.attname for f in fields
if not getattr(f, '_lino_babel_field', False)
])))
if model._meta.parents:
if len(model._meta.parents) != 1:
msg = "%s : model._meta.parents is %r" % (
model, model._meta.parents)
raise Exception(msg)
pm, pf = model._meta.parents.items()[0]
child_fields = [f for f in fields if f != pf]
if child_fields:
attrs = ',' + ','.join([
'%s=%s' % (f.attname, f.attname) for f in child_fields
])
else:
attrs = ''
#~ self.stream.write(' return insert_child(%s.objects.get(pk=%s),%s%s)\n' % (
#~ full_model_name(pm,'_'),pf.attname,full_model_name(model,'_'),attrs))
self.stream.write(' return create_mti_child(%s,%s,%s%s)\n' %
(full_model_name(pm, '_'), pf.attname,
full_model_name(model, '_'), attrs))
else:
self.stream.write(" kw = dict()\n")
for f in fields:
if getattr(f, '_lino_babel_field', False):
continue
elif isinstance(f, (BabelCharField, BabelTextField)):
self.stream.write(
' if %s is not None: kw.update(bv2kw(%r,%s))\n'
% (f.attname, f.attname, f.attname))
else:
if isinstance(f, models.DecimalField):
self.stream.write(
' if %s is not None: %s = Decimal(%s)\n' %
(f.attname, f.attname, f.attname))
elif isinstance(
f,
models.ForeignKey) and f.rel.to is ContentType:
#~ self.stream.write(
#~ ' %s = ContentType.objects.get_for_model(%s).pk\n' % (
#~ f.attname,f.attname))
self.stream.write(
' %s = new_content_type_id(%s)\n' %
(f.attname, f.attname))
self.stream.write(' kw.update(%s=%s)\n' %
(f.attname, f.attname))
self.stream.write(' return %s(**kw)\n\n' %
full_model_name(model, '_'))
self.stream.write('\n')
#~ used_models = set()
self.stream.write("""
def main():
loader = DpyLoader(globals())
from django.core.management import call_command
# call_command('initdb', interactive=False)
call_command('initdb')
os.chdir(os.path.dirname(__file__))
loader.initialize()
""")
for model in progress.bar(self.models):
filename = '%s.py' % model._meta.db_table
filename = os.path.join(self.output_dir, filename)
# puts("Writing {0}...".format(filename))
stream = file(filename, 'wt')
stream.write('# -*- coding: UTF-8 -*-\n')
qs = model.objects.all()
try:
stream.write(
'logger.info("Loading %d objects to table %s...")\n' %
(qs.count(), model._meta.db_table))
fields = [
f for f, m in model._meta.get_fields_with_model()
if m is None
]
fields = [
f for f in fields
if not getattr(f, '_lino_babel_field', False)
]
if AFTER17:
fields = [f for f in fields if f.concrete]
stream.write("# fields: %s\n" %
', '.join([f.name for f in fields]))
for obj in qs:
self.count_objects += 1
#~ used_models.add(model)
stream.write(
'loader.save(create_%s(%s))\n' %
(obj._meta.db_table, ','.join(
[self.value2string(obj, f) for f in fields])))
stream.write('\n')
stream.write('loader.flush_deferred_objects()\n')
except ProgrammingError as e:
if not self.options['tolerate']:
raise
stream.write('\n')
logger.warning("Tolerating database error %s in %s", e,
model._meta.db_table)
msg = ("The data of this table has not been dumped"
"because an error {0} occured.").format(e)
stream.write('raise Exception("{0}")\n'.format(msg))
stream.close()
#~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % )
self.stream.write(' execfile("%s.py")\n' % model._meta.db_table)
self.stream.write(' loader.finalize()\n')
# self.stream.write(
# ' logger.info("Loaded %d objects",loader.count_objects)\n')
self.stream.write("""
if __name__ == '__main__':
main()
""")
#~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n')
self.stream.close()
def write_files(self):
puts("Writing {0}...".format(self.main_file))
self.stream = open(self.main_file, 'wt')
current_version = settings.SITE.version
self.stream.write('''\
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# This is a Python dump created using dump2py.
# DJANGO_SETTINGS_MODULE was %r, TIME_ZONE was %r.
''' % (settings.SETTINGS_MODULE, settings.TIME_ZONE))
self.stream.write('''
from __future__ import unicode_literals
import logging
logger = logging.getLogger('%s')
''' % __name__)
self.stream.write('SOURCE_VERSION = %r\n' % str(current_version))
self.stream.write('''
import os
import six
from decimal import Decimal
from datetime import datetime
from datetime import time, date
from django.conf import settings
from django.utils.timezone import make_aware, utc
from django.core.management import call_command
# from django.contrib.contenttypes.models import ContentType
from lino.utils.dpy import create_mti_child
from lino.utils.dpy import DpyLoader
from lino.core.utils import resolve_model
if settings.USE_TZ:
def dt(*args):
return make_aware(datetime(*args), timezone=utc)
else:
def dt(*args):
return datetime(*args)
def new_content_type_id(m):
if m is None: return m
ct = settings.SITE.models.contenttypes.ContentType.objects.get_for_model(m)
if ct is None: return None
return ct.pk
def pmem():
# Thanks to https://stackoverflow.com/questions/938733/total-memory-used-by-python-process
process = psutil.Process(os.getpid())
print(process.memory_info().rss)
def execfile(fn, *args):
logger.info("Execute file %s ...", fn)
six.exec_(compile(open(fn, "rb").read(), fn, 'exec'), *args)
# pmem() # requires pip install psutil
''')
s = ','.join([
'%s=values[%d]' % (lng.name, lng.index)
for lng in settings.SITE.languages
])
self.stream.write('''
def bv2kw(fieldname, values):
"""
Needed if `Site.languages` changed between dumpdata and loaddata
"""
return settings.SITE.babelkw(fieldname, %s)
''' % s)
self.models = sorted_models_list()
if settings.SITE.is_installed('contenttypes'):
from django.contrib.contenttypes.models import ContentType
self.models = [
m for m in self.models if not issubclass(m, ContentType)
]
if settings.SITE.is_installed('sessions'):
from django.contrib.sessions.models import Session
self.models = [
m for m in self.models if not issubclass(m, Session)
]
for model in self.models:
self.stream.write(
'%s = resolve_model("%s")\n' %
(full_model_name(model, '_'), full_model_name(model)))
self.stream.write('\n')
self.models = self.sort_models(self.models)
self.stream.write('\n')
for model in self.models:
write_create_function(model, self.stream)
self.stream.write('\n')
#~ used_models = set()
self.stream.write("""
def main(args):
loader = DpyLoader(globals(), quick=args.quick)
from django.core.management import call_command
call_command('initdb', interactive=args.interactive)
os.chdir(os.path.dirname(__file__))
loader.initialize()
args = (globals(), locals())
""")
max_row_count = self.options['max_row_count']
for model in progress.bar(self.models):
try:
qs = model.objects.all()
total_count = qs.count()
except DatabaseError as e:
self.database_errors += 1
if not self.options['tolerate']:
raise
self.stream.write('\n')
logger.warning("Tolerating database error %s in %s", e,
model._meta.db_table)
msg = ("The data of table {0} has not been dumped"
"because an error {1} occured.").format(
model._meta.db_table, e)
self.stream.write('raise Exception("{0}")\n'.format(msg))
continue
fields = [
f for f in model._meta.get_fields()
if f.concrete and f.model is model
]
fields = [
f for f in fields if not getattr(f, '_lino_babel_field', False)
]
chunks = [] # list of tuples (i, filename, queryset)
if total_count > max_row_count:
num_files = (total_count // max_row_count) + 1
for i in range(num_files):
o1 = max_row_count * i
o2 = max_row_count * (i + 1)
t = (i + 1, '%s_%d.py' % (model._meta.db_table, i + 1),
qs[o1:o2])
chunks.append(t)
else:
chunks.append((1, '%s.py' % model._meta.db_table, qs))
for i, filename, qs in chunks:
self.stream.write(' execfile("%s", *args)\n' % filename)
filename = os.path.join(self.output_dir, filename)
# puts("Writing {0}...".format(filename))
# stream = file(filename, 'wt')
stream = open(filename, 'wt')
stream.write('# -*- coding: UTF-8 -*-\n')
txt = "%d objects" % total_count
if len(chunks) > 1:
txt += " (part %d of %d)" % (i, len(chunks))
stream.write('logger.info("Loading %s to table %s...")\n' %
(txt, model._meta.db_table))
stream.write("# fields: %s\n" %
', '.join([f.name for f in fields]))
for obj in qs:
self.count_objects += 1
#~ used_models.add(model)
stream.write(
'loader.save(create_%s(%s))\n' %
(obj._meta.db_table, ','.join(
[self.value2string(obj, f) for f in fields])))
stream.write('\n')
if i == len(chunks):
stream.write('loader.flush_deferred_objects()\n')
stream.close()
#~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % )
self.stream.write(' loader.finalize()\n')
# 20180416 why was the following message commented out?
# reactivated it because otherwise we have no log entry when
# the process has finished.
self.stream.write(
' logger.info("Loaded %d objects", loader.count_objects)\n')
self.stream.write(" call_command('resetsequences')\n")
self.stream.write("""
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='Restore the data.')
parser.add_argument('--noinput', dest='interactive',
action='store_false', default=True,
help="Don't ask for confirmation before flushing the database.")
parser.add_argument('--quick', dest='quick',
action='store_true',default=False,
help='Do not call full_clean() on restored instances.')
args = parser.parse_args()
main(args)
""")
#~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n')
self.stream.close()
def write_files(self):
puts("Writing {0}...".format(self.main_file))
self.stream = open(self.main_file, 'wt')
current_version = settings.SITE.version
self.stream.write('''\
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# This is a Python dump created using dump2py.
# DJANGO_SETTINGS_MODULE was %r, TIME_ZONE was %r.
''' % (settings.SETTINGS_MODULE, settings.TIME_ZONE))
self.stream.write('''
from __future__ import unicode_literals
import logging
logger = logging.getLogger('%s')
import os
''' % __name__)
self.stream.write('SOURCE_VERSION = %r\n' % str(current_version))
self.stream.write('''
from decimal import Decimal
from datetime import datetime
from datetime import time, date
from django.conf import settings
from django.utils.timezone import make_aware
# from django.contrib.contenttypes.models import ContentType
from lino.utils.dpy import create_mti_child
from lino.utils.dpy import DpyLoader
from lino.core.utils import resolve_model
if settings.USE_TZ:
def dt(*args):
return make_aware(datetime(*args))
else:
def dt(*args):
return datetime(*args)
def new_content_type_id(m):
if m is None: return m
ct = settings.SITE.models.contenttypes.ContentType.objects.get_for_model(m)
if ct is None: return None
return ct.pk
''')
s = ','.join([
'%s=values[%d]' % (lng.name, lng.index)
for lng in settings.SITE.languages])
self.stream.write('''
def bv2kw(fieldname, values):
"""
Needed if `Site.languages` changed between dumpdata and loaddata
"""
return settings.SITE.babelkw(fieldname, %s)
''' % s)
self.models = sorted_models_list()
if settings.SITE.is_installed('contenttypes'):
from django.contrib.contenttypes.models import ContentType
self.models = [m for m in self.models
if not issubclass(m, ContentType)]
if settings.SITE.is_installed('sessions'):
from django.contrib.sessions.models import Session
self.models = [m for m in self.models
if not issubclass(m, Session)]
for model in self.models:
self.stream.write('%s = resolve_model("%s")\n' % (
full_model_name(model, '_'), full_model_name(model)))
self.stream.write('\n')
self.models = self.sort_models(self.models)
self.stream.write('\n')
for model in self.models:
fields = [f for f in model._meta.get_fields()
if f.concrete and f.model is model]
for f in fields:
if getattr(f, 'auto_now_add', False):
raise Exception("%s.%s.auto_now_add is True : values will be lost!" % (
full_model_name(model), f.name))
#~ fields = model._meta.local_fields
#~ fields = [f for f in model._meta.fields if f.serialize]
#~ fields = [f for f in model._meta.local_fields if f.serialize]
self.stream.write('def create_%s(%s):\n' % (
model._meta.db_table, ', '.join([
f.attname for f in fields
if not getattr(f, '_lino_babel_field', False)])))
for f in fields:
if isinstance(f, models.DecimalField):
self.stream.write(
' if %s is not None: %s = Decimal(%s)\n' % (
f.attname, f.attname, f.attname))
elif isinstance(f, ChoiceListField):
lstname = 'settings.SITE.modules.{0}.{1}'.format(
f.choicelist.app_label, f.choicelist.__name__)
ln = ' if {0}: {0} = {1}.get_by_value({0})\n'
self.stream.write(ln.format(f.attname, lstname))
elif is_pointer_to_contenttype(f):
self.stream.write(
' %s = new_content_type_id(%s)\n' % (
f.attname, f.attname))
if model._meta.parents:
if len(model._meta.parents) != 1:
msg = "%s : model._meta.parents is %r" % (
model, model._meta.parents)
raise Exception(msg)
pm, pf = list(model._meta.parents.items())[0]
child_fields = [f for f in fields if f != pf]
if child_fields:
attrs = ',' + ','.join([
'%s=%s' % (f.attname, f.attname)
for f in child_fields])
else:
attrs = ''
self.stream.write(
' return create_mti_child(%s, %s, %s%s)\n' % (
full_model_name(pm, '_'), pf.attname,
full_model_name(model, '_'), attrs))
else:
self.stream.write(" kw = dict()\n")
for f in fields:
if getattr(f, '_lino_babel_field', False):
continue
elif isinstance(f, (BabelCharField, BabelTextField)):
self.stream.write(
' if %s is not None: kw.update(bv2kw(%r,%s))\n' % (
f.attname, f.attname, f.attname))
else:
self.stream.write(
' kw.update(%s=%s)\n' % (f.attname, f.attname))
self.stream.write(' return %s(**kw)\n\n' %
full_model_name(model, '_'))
self.stream.write('\n')
#~ used_models = set()
self.stream.write("""
def main(args):
loader = DpyLoader(globals())
from django.core.management import call_command
call_command('initdb', interactive=args.interactive)
os.chdir(os.path.dirname(__file__))
loader.initialize()
""")
for model in progress.bar(self.models):
filename = '%s.py' % model._meta.db_table
filename = os.path.join(self.output_dir, filename)
# puts("Writing {0}...".format(filename))
# stream = file(filename, 'wt')
stream = open(filename, 'wt')
stream.write('# -*- coding: UTF-8 -*-\n')
qs = model.objects.all()
try:
stream.write(
'logger.info("Loading %d objects to table %s...")\n' % (
qs.count(), model._meta.db_table))
fields = [f for f in model._meta.get_fields()
if f.concrete and f.model is model]
fields = [
f for f in fields
if not getattr(f, '_lino_babel_field', False)]
stream.write(
"# fields: %s\n" % ', '.join(
[f.name for f in fields]))
for obj in qs:
self.count_objects += 1
#~ used_models.add(model)
stream.write('loader.save(create_%s(%s))\n' % (
obj._meta.db_table,
','.join([self.value2string(obj, f) for f in fields])))
stream.write('\n')
stream.write('loader.flush_deferred_objects()\n')
except DatabaseError as e:
self.database_errors += 1
if not self.options['tolerate']:
raise
stream.write('\n')
logger.warning("Tolerating database error %s in %s",
e, model._meta.db_table)
msg = ("The data of this table has not been dumped"
"because an error {0} occured.").format(e)
stream.write('raise Exception("{0}")\n'.format(msg))
stream.close()
#~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % )
self.stream.write(' execfile("%s.py")\n' % model._meta.db_table)
self.stream.write(
' loader.finalize()\n')
# self.stream.write(
# ' logger.info("Loaded %d objects",loader.count_objects)\n')
self.stream.write("""
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='Restore the data.')
parser.add_argument('--noinput', dest='interactive',
action='store_false', default=True,
help="Don't ask for confirmation before flushing the database.")
args = parser.parse_args()
main(args)
""")
#~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n')
self.stream.close()
def serialize(self, queryset, **options):
from django.contrib.sessions.models import Session
from django.contrib.contenttypes.models import ContentType
self.options = options
self.stream = options.get("stream", StringIO())
self.selected_fields = options.get("fields")
self.use_natural_keys = options.get("use_natural_keys", False)
if self.write_preamble:
current_version = settings.SITE.version
if '+' in current_version:
logger.warning(
"Dumpdata from intermediate version %s" % current_version)
self.stream.write('# -*- coding: UTF-8 -*-\n')
self.stream.write('''\
"""
This is a `Python dump created using %s.
''' % settings.SITE.using_text())
#~ self.stream.write(settings.SITE.welcome_text())
self.stream.write('''
"""
from __future__ import unicode_literals
''')
self.stream.write('SOURCE_VERSION = %r\n' % str(current_version))
self.stream.write('from decimal import Decimal\n')
self.stream.write('from datetime import datetime as dt\n')
self.stream.write('from datetime import time,date\n')
self.stream.write('from lino.utils.dpy import create_mti_child\n')
self.stream.write('from lino.utils.dbutils import resolve_model\n')
self.stream.write(
'from lino.modlib.gfks.models import ContentType\n')
self.stream.write('from django.conf import settings\n')
self.stream.write('''
def new_content_type_id(m):
if m is None: return m
# if not fmn: return None
# m = resolve_model(fmn)
ct = ContentType.objects.get_for_model(m)
if ct is None: return None
return ct.pk
''')
#~ s = ','.join([
#~ '%s=values[%d]' % (k,i)
#~ for i,k in enumerate(settings.SITE.AVAILABLE_LANGUAGES)])
s = ','.join([
'%s=values[%d]' % (lng.name, lng.index)
for lng in settings.SITE.languages])
self.stream.write('''
def bv2kw(fieldname,values):
"""
Needed if `Site.languages` changed between dumpdata and loaddata
"""
return settings.SITE.babelkw(fieldname,%s)
''' % s)
#~ model = queryset.model
if self.models is None:
self.models = sorted_models_list() # models.get_models()
if self.write_preamble:
for model in self.models:
self.stream.write('%s = resolve_model("%s")\n' % (
full_model_name(model, '_'), full_model_name(model)))
self.stream.write('\n')
for model in self.models:
fields = [f for f,
m in model._meta.get_fields_with_model() if m is None]
for f in fields:
if getattr(f, 'auto_now_add', False):
raise Exception("%s.%s.auto_now_add is True : values will be lost!" % (
full_model_name(model), f.name))
field_names = [f.attname for f in fields
if not getattr(f, '_lino_babel_field', False)]
self.stream.write('def create_%s(%s):\n' % (
model._meta.db_table, ', '.join(field_names)))
if model._meta.parents:
if len(model._meta.parents) != 1:
msg = "%s : model._meta.parents is %r" % (
model, model._meta.parents)
raise Exception(msg)
pm, pf = list(model._meta.parents.items())[0]
child_fields = [f for f in fields if f != pf]
if child_fields:
attrs = ', ' + ', '.join([
'%s=%s' % (f.attname, f.attname)
for f in child_fields])
else:
attrs = ''
tpl = ' return create_mti_child(%s, %s, %s%s)\n'
self.stream.write(tpl % (
full_model_name(pm, '_'),
pf.attname, full_model_name(model, '_'), attrs))
else:
self.stream.write(" kw = dict()\n")
for f in fields:
if getattr(f, '_lino_babel_field', False):
continue
elif isinstance(f, (BabelCharField, BabelTextField)):
tpl = ' if %s is not None:'
tpl += ' kw.update(bv2kw(%r, %s))\n'
self.stream.write(
tpl % (f.attname, f.attname, f.attname))
else:
if isinstance(f, models.DecimalField):
self.stream.write(
' if %s is not None: %s = Decimal(%s)\n' % (
f.attname, f.attname, f.attname))
elif isinstance(f, ChoiceListField):
lstname = 'settings.SITE.modules.{0}.{1}'.format(
f.choicelist.app_label, f.choicelist.__name__)
ln = ' if {0}: {0} = {1}.get_by_value({0})\n'
self.stream.write(ln.format(f.attname, lstname))
elif isinstance(f, ForeignKey):
if f.rel.model is ContentType:
self.stream.write(
' %s = new_content_type_id(%s)\n' % (
f.attname, f.attname))
self.stream.write(
' kw.update(%s=%s)\n' % (f.attname, f.attname))
self.stream.write(' return %s(**kw)\n\n' %
full_model_name(model, '_'))
#~ self.start_serialization()
self.stream.write('\n')
model = None
all_models = []
for obj in queryset:
if isinstance(obj, ContentType):
continue
if isinstance(obj, Session):
continue
if obj.__class__ != model:
model = obj.__class__
if model in all_models:
raise Exception("%s instances weren't grouped!" % model)
all_models.append(model)
self.stream.write('\ndef %s_objects():\n' %
model._meta.db_table)
fields = [f for f,
m in model._meta.get_fields_with_model() if m is None]
fields = [
f for f in fields if not getattr(f, '_lino_babel_field', False)]
self.stream.write(' yield create_%s(%s)\n' % (
obj._meta.db_table,
','.join([self.value2string(obj, f) for f in fields])))
self.stream.write('\n\ndef objects():\n')
all_models = self.sort_models(all_models)
for model in all_models:
#~ self.stream.write(' for o in %s_objects(): yield o\n' % model._meta.db_table)
self.stream.write(' yield %s_objects()\n' %
model._meta.db_table)
def write_files(self):
puts("Writing {0}...".format(self.main_file))
self.stream = open(self.main_file, 'wt')
current_version = settings.SITE.version
self.stream.write('''\
#!/usr/bin/env python
# -*- coding: UTF-8 -*-
# This is a Python dump created using dump2py.
# DJANGO_SETTINGS_MODULE was %r, TIME_ZONE was %r.
''' % (settings.SETTINGS_MODULE, settings.TIME_ZONE))
self.stream.write('''
from __future__ import unicode_literals
import logging
logger = logging.getLogger('%s')
''' % __name__)
self.stream.write('SOURCE_VERSION = %r\n' % str(current_version))
self.stream.write('''
import os
import six
from decimal import Decimal
from datetime import datetime
from datetime import time, date
from django.conf import settings
from django.utils.timezone import make_aware, utc
from django.core.management import call_command
# from django.contrib.contenttypes.models import ContentType
from lino.utils.dpy import create_mti_child
from lino.utils.dpy import DpyLoader
from lino.core.utils import resolve_model
if settings.USE_TZ:
def dt(*args):
return make_aware(datetime(*args), timezone=utc)
else:
def dt(*args):
return datetime(*args)
def new_content_type_id(m):
if m is None: return m
ct = settings.SITE.models.contenttypes.ContentType.objects.get_for_model(m)
if ct is None: return None
return ct.pk
def pmem():
# Thanks to https://stackoverflow.com/questions/938733/total-memory-used-by-python-process
process = psutil.Process(os.getpid())
print(process.memory_info().rss)
def execfile(fn, *args):
logger.info("Execute file %s ...", fn)
six.exec_(compile(open(fn, "rb").read(), fn, 'exec'), *args)
# pmem() # requires pip install psutil
''')
s = ','.join([
'%s=values[%d]' % (lng.name, lng.index)
for lng in settings.SITE.languages])
self.stream.write('''
def bv2kw(fieldname, values):
"""
Needed if `Site.languages` changed between dumpdata and loaddata
"""
return settings.SITE.babelkw(fieldname, %s)
''' % s)
self.models = sorted_models_list()
if settings.SITE.is_installed('contenttypes'):
from django.contrib.contenttypes.models import ContentType
self.models = [m for m in self.models
if not issubclass(m, ContentType)]
if settings.SITE.is_installed('sessions'):
from django.contrib.sessions.models import Session
self.models = [m for m in self.models
if not issubclass(m, Session)]
for model in self.models:
self.stream.write('%s = resolve_model("%s")\n' % (
full_model_name(model, '_'), full_model_name(model)))
self.stream.write('\n')
self.models = self.sort_models(self.models)
self.stream.write('\n')
for model in self.models:
write_create_function(model, self.stream)
self.stream.write('\n')
#~ used_models = set()
self.stream.write("""
def main(args):
loader = DpyLoader(globals(), quick=args.quick)
from django.core.management import call_command
call_command('initdb', interactive=args.interactive)
os.chdir(os.path.dirname(__file__))
loader.initialize()
args = (globals(), locals())
""")
max_row_count = self.options['max_row_count']
for model in progress.bar(self.models):
try:
qs = model.objects.all()
total_count = qs.count()
except DatabaseError as e:
self.database_errors += 1
if not self.options['tolerate']:
raise
self.stream.write('\n')
logger.warning("Tolerating database error %s in %s",
e, model._meta.db_table)
msg = ("The data of table {0} has not been dumped"
"because an error {1} occured.").format(
model._meta.db_table, e)
self.stream.write('raise Exception("{0}")\n'.format(msg))
continue
fields = [f for f in model._meta.get_fields()
if f.concrete and f.model is model]
fields = [
f for f in fields
if not getattr(f, '_lino_babel_field', False)]
chunks = [] # list of tuples (i, filename, queryset)
if total_count > max_row_count:
num_files = (total_count // max_row_count) + 1
for i in range(num_files):
o1 = max_row_count * i
o2 = max_row_count * (i+1)
t = (i+1,
'%s_%d.py' % (model._meta.db_table, i+1),
qs[o1:o2])
chunks.append(t)
else:
chunks.append((1, '%s.py' % model._meta.db_table, qs))
for i, filename, qs in chunks:
self.stream.write(' execfile("%s", *args)\n' % filename)
filename = os.path.join(self.output_dir, filename)
# puts("Writing {0}...".format(filename))
# stream = file(filename, 'wt')
stream = open(filename, 'wt')
stream.write('# -*- coding: UTF-8 -*-\n')
txt = "%d objects" % total_count
if len(chunks) > 1:
txt += " (part %d of %d)" % (i, len(chunks))
stream.write(
'logger.info("Loading %s to table %s...")\n' % (
txt, model._meta.db_table))
stream.write(
"# fields: %s\n" % ', '.join(
[f.name for f in fields]))
for obj in qs:
self.count_objects += 1
#~ used_models.add(model)
stream.write('loader.save(create_%s(%s))\n' % (
obj._meta.db_table,
','.join([self.value2string(obj, f) for f in fields])))
stream.write('\n')
if i == len(chunks):
stream.write('loader.flush_deferred_objects()\n')
stream.close()
#~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % )
self.stream.write(
' loader.finalize()\n')
# 20180416 why was the following message commented out?
# reactivated it because otherwise we have no log entry when
# the process has finished.
self.stream.write(
' logger.info("Loaded %d objects", loader.count_objects)\n')
self.stream.write(
" call_command('resetsequences')\n")
self.stream.write("""
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='Restore the data.')
parser.add_argument('--noinput', dest='interactive',
action='store_false', default=True,
help="Don't ask for confirmation before flushing the database.")
parser.add_argument('--quick', dest='quick',
action='store_true',default=False,
help='Do not call full_clean() on restored instances.')
args = parser.parse_args()
main(args)
""")
#~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n')
self.stream.close()
