def write_files(self): puts("Writing {0}...".format(self.main_file)) self.stream = open(self.main_file, "wt") current_version = settings.SITE.version self.stream.write( """\ #!/usr/bin/env python # -*- coding: UTF-8 -*- """ ) self.stream.write( '''\ """ This is a Python dump created using %s. ''' % settings.SITE.using_text() ) # ~ self.stream.write(settings.SITE.welcome_text()) self.stream.write( ''' """ from __future__ import unicode_literals import logging logger = logging.getLogger('%s') import os ''' % __name__ ) if False: self.stream.write( """ os.environ['DJANGO_SETTINGS_MODULE'] = '%s' """ % settings.SETTINGS_MODULE ) self.stream.write("SOURCE_VERSION = %r\n" % str(current_version)) self.stream.write( """ from decimal import Decimal from datetime import datetime as dt from datetime import time, date from django.conf import settings # from django.contrib.contenttypes.models import ContentType from lino.utils.dpy import create_mti_child from lino.utils.dpy import DpyLoader from lino.core.utils import resolve_model def new_content_type_id(m): if m is None: return m ct = settings.SITE.modules.contenttypes.ContentType.objects.get_for_model(m) if ct is None: return None return ct.pk """ ) s = ",".join(["%s=values[%d]" % (lng.name, lng.index) for lng in settings.SITE.languages]) self.stream.write( ''' def bv2kw(fieldname, values): """ Needed if `Site.languages` changed between dumpdata and loaddata """ return settings.SITE.babelkw(fieldname, %s) ''' % s ) self.models = sorted_models_list() if settings.SITE.is_installed("contenttypes"): from django.contrib.contenttypes.models import ContentType self.models = [m for m in self.models if not issubclass(m, ContentType)] if settings.SITE.is_installed("sessions"): from django.contrib.sessions.models import Session self.models = [m for m in self.models if not issubclass(m, Session)] for model in self.models: self.stream.write('%s = resolve_model("%s")\n' % (full_model_name(model, "_"), full_model_name(model))) self.stream.write("\n") self.models = self.sort_models(self.models) self.stream.write("\n") for model in self.models: if AFTER17: fields = [f for f in model._meta.get_fields() if f.concrete and f.model is model] else: fields = [f for f, m in model._meta.get_fields_with_model() if m is None] for f in fields: if getattr(f, "auto_now_add", False): raise Exception( "%s.%s.auto_now_add is True : values will be lost!" % (full_model_name(model), f.name) ) # ~ fields = model._meta.local_fields # ~ fields = [f for f in model._meta.fields if f.serialize] # ~ fields = [f for f in model._meta.local_fields if f.serialize] self.stream.write( "def create_%s(%s):\n" % ( model._meta.db_table, ", ".join([f.attname for f in fields if not getattr(f, "_lino_babel_field", False)]), ) ) for f in fields: if isinstance(f, models.DecimalField): self.stream.write(" if %s is not None: %s = Decimal(%s)\n" % (f.attname, f.attname, f.attname)) elif isinstance(f, ChoiceListField): lstname = "settings.SITE.modules.{0}.{1}".format(f.choicelist.app_label, f.choicelist.__name__) ln = " if {0}: {0} = {1}.get_by_value({0})\n" self.stream.write(ln.format(f.attname, lstname)) elif is_pointer_to_contenttype(f): self.stream.write(" %s = new_content_type_id(%s)\n" % (f.attname, f.attname)) if model._meta.parents: if len(model._meta.parents) != 1: msg = "%s : model._meta.parents is %r" % (model, model._meta.parents) raise Exception(msg) pm, pf = list(model._meta.parents.items())[0] child_fields = [f for f in fields if f != pf] if child_fields: attrs = "," + ",".join(["%s=%s" % (f.attname, f.attname) for f in child_fields]) else: attrs = "" self.stream.write( " return create_mti_child(%s, %s, %s%s)\n" % (full_model_name(pm, "_"), pf.attname, full_model_name(model, "_"), attrs) ) else: self.stream.write(" kw = dict()\n") for f in fields: if getattr(f, "_lino_babel_field", False): continue elif isinstance(f, (BabelCharField, BabelTextField)): self.stream.write( " if %s is not None: kw.update(bv2kw(%r,%s))\n" % (f.attname, f.attname, f.attname) ) else: self.stream.write(" kw.update(%s=%s)\n" % (f.attname, f.attname)) self.stream.write(" return %s(**kw)\n\n" % full_model_name(model, "_")) self.stream.write("\n") # ~ used_models = set() self.stream.write( """ def main(): loader = DpyLoader(globals()) from django.core.management import call_command # call_command('initdb', interactive=False) call_command('initdb') os.chdir(os.path.dirname(__file__)) loader.initialize() """ ) for model in progress.bar(self.models): filename = "%s.py" % model._meta.db_table filename = os.path.join(self.output_dir, filename) # puts("Writing {0}...".format(filename)) stream = file(filename, "wt") stream.write("# -*- coding: UTF-8 -*-\n") qs = model.objects.all() try: stream.write('logger.info("Loading %d objects to table %s...")\n' % (qs.count(), model._meta.db_table)) if AFTER17: fields = [f for f in model._meta.get_fields() if f.concrete and f.model is model] else: fields = [f for f, m in model._meta.get_fields_with_model() if m is None] fields = [f for f in fields if not getattr(f, "_lino_babel_field", False)] stream.write("# fields: %s\n" % ", ".join([f.name for f in fields])) for obj in qs: self.count_objects += 1 # ~ used_models.add(model) stream.write( "loader.save(create_%s(%s))\n" % (obj._meta.db_table, ",".join([self.value2string(obj, f) for f in fields])) ) stream.write("\n") stream.write("loader.flush_deferred_objects()\n") except ProgrammingError as e: if not self.options["tolerate"]: raise stream.write("\n") logger.warning("Tolerating database error %s in %s", e, model._meta.db_table) msg = ("The data of this table has not been dumped" "because an error {0} occured.").format(e) stream.write('raise Exception("{0}")\n'.format(msg)) stream.close() # ~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % ) self.stream.write(' execfile("%s.py")\n' % model._meta.db_table) self.stream.write(" loader.finalize()\n") # self.stream.write( # ' logger.info("Loaded %d objects",loader.count_objects)\n') self.stream.write( """ if __name__ == '__main__': main() """ ) # ~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n') self.stream.close()
def write_files(self): puts("Writing {0}...".format(self.main_file)) self.stream = open(self.main_file, 'wt') current_version = settings.SITE.version self.stream.write('''\ #!/usr/bin/env python # -*- coding: UTF-8 -*- # This is a Python dump created using dump2py. # DJANGO_SETTINGS_MODULE was %r, TIME_ZONE was %r. ''' % (settings.SETTINGS_MODULE, settings.TIME_ZONE)) self.stream.write(''' from __future__ import unicode_literals import logging logger = logging.getLogger('%s') import os ''' % __name__) self.stream.write('SOURCE_VERSION = %r\n' % str(current_version)) self.stream.write(''' from decimal import Decimal from datetime import datetime from datetime import time, date from django.conf import settings from django.utils.timezone import make_aware # from django.contrib.contenttypes.models import ContentType from lino.utils.dpy import create_mti_child from lino.utils.dpy import DpyLoader from lino.core.utils import resolve_model if settings.USE_TZ: def dt(*args): return make_aware(datetime(*args)) else: def dt(*args): return datetime(*args) def new_content_type_id(m): if m is None: return m ct = settings.SITE.models.contenttypes.ContentType.objects.get_for_model(m) if ct is None: return None return ct.pk ''') s = ','.join([ '%s=values[%d]' % (lng.name, lng.index) for lng in settings.SITE.languages ]) self.stream.write(''' def bv2kw(fieldname, values): """ Needed if `Site.languages` changed between dumpdata and loaddata """ return settings.SITE.babelkw(fieldname, %s) ''' % s) self.models = sorted_models_list() if settings.SITE.is_installed('contenttypes'): from django.contrib.contenttypes.models import ContentType self.models = [ m for m in self.models if not issubclass(m, ContentType) ] if settings.SITE.is_installed('sessions'): from django.contrib.sessions.models import Session self.models = [ m for m in self.models if not issubclass(m, Session) ] for model in self.models: self.stream.write( '%s = resolve_model("%s")\n' % (full_model_name(model, '_'), full_model_name(model))) self.stream.write('\n') self.models = self.sort_models(self.models) self.stream.write('\n') for model in self.models: fields = [ f for f in model._meta.get_fields() if f.concrete and f.model is model ] for f in fields: if getattr(f, 'auto_now_add', False): raise Exception( "%s.%s.auto_now_add is True : values will be lost!" % (full_model_name(model), f.name)) #~ fields = model._meta.local_fields #~ fields = [f for f in model._meta.fields if f.serialize] #~ fields = [f for f in model._meta.local_fields if f.serialize] self.stream.write('def create_%s(%s):\n' % (model._meta.db_table, ', '.join([ f.attname for f in fields if not getattr(f, '_lino_babel_field', False) ]))) for f in fields: if isinstance(f, models.DecimalField): self.stream.write( ' if %s is not None: %s = Decimal(%s)\n' % (f.attname, f.attname, f.attname)) elif isinstance(f, ChoiceListField): lstname = 'settings.SITE.modules.{0}.{1}'.format( f.choicelist.app_label, f.choicelist.__name__) ln = ' if {0}: {0} = {1}.get_by_value({0})\n' self.stream.write(ln.format(f.attname, lstname)) elif is_pointer_to_contenttype(f): self.stream.write(' %s = new_content_type_id(%s)\n' % (f.attname, f.attname)) if model._meta.parents: if len(model._meta.parents) != 1: msg = "%s : model._meta.parents is %r" % ( model, model._meta.parents) raise Exception(msg) pm, pf = list(model._meta.parents.items())[0] child_fields = [f for f in fields if f != pf] if child_fields: attrs = ',' + ','.join([ '%s=%s' % (f.attname, f.attname) for f in child_fields ]) else: attrs = '' self.stream.write( ' return create_mti_child(%s, %s, %s%s)\n' % (full_model_name(pm, '_'), pf.attname, full_model_name(model, '_'), attrs)) else: self.stream.write(" kw = dict()\n") for f in fields: if getattr(f, '_lino_babel_field', False): continue elif isinstance(f, (BabelCharField, BabelTextField)): self.stream.write( ' if %s is not None: kw.update(bv2kw(%r,%s))\n' % (f.attname, f.attname, f.attname)) else: self.stream.write(' kw.update(%s=%s)\n' % (f.attname, f.attname)) self.stream.write(' return %s(**kw)\n\n' % full_model_name(model, '_')) self.stream.write('\n') #~ used_models = set() self.stream.write(""" def main(args): loader = DpyLoader(globals()) from django.core.management import call_command call_command('initdb', interactive=args.interactive) os.chdir(os.path.dirname(__file__)) loader.initialize() """) for model in progress.bar(self.models): filename = '%s.py' % model._meta.db_table filename = os.path.join(self.output_dir, filename) # puts("Writing {0}...".format(filename)) # stream = file(filename, 'wt') stream = open(filename, 'wt') stream.write('# -*- coding: UTF-8 -*-\n') qs = model.objects.all() try: stream.write( 'logger.info("Loading %d objects to table %s...")\n' % (qs.count(), model._meta.db_table)) fields = [ f for f in model._meta.get_fields() if f.concrete and f.model is model ] fields = [ f for f in fields if not getattr(f, '_lino_babel_field', False) ] stream.write("# fields: %s\n" % ', '.join([f.name for f in fields])) for obj in qs: self.count_objects += 1 #~ used_models.add(model) stream.write( 'loader.save(create_%s(%s))\n' % (obj._meta.db_table, ','.join( [self.value2string(obj, f) for f in fields]))) stream.write('\n') stream.write('loader.flush_deferred_objects()\n') except DatabaseError as e: self.database_errors += 1 if not self.options['tolerate']: raise stream.write('\n') logger.warning("Tolerating database error %s in %s", e, model._meta.db_table) msg = ("The data of this table has not been dumped" "because an error {0} occured.").format(e) stream.write('raise Exception("{0}")\n'.format(msg)) stream.close() #~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % ) self.stream.write(' execfile("%s.py")\n' % model._meta.db_table) self.stream.write(' loader.finalize()\n') # self.stream.write( # ' logger.info("Loaded %d objects",loader.count_objects)\n') self.stream.write(""" if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description='Restore the data.') parser.add_argument('--noinput', dest='interactive', action='store_false', default=True, help="Don't ask for confirmation before flushing the database.") args = parser.parse_args() main(args) """) #~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n') self.stream.close()
def write_files(self): puts("Writing {0}...".format(self.main_file)) self.stream = open(self.main_file, 'wt') current_version = settings.SITE.version self.stream.write("""\ #!/usr/bin/env python # -*- coding: UTF-8 -*- """) self.stream.write('''\ """ This is a Python dump created using %s. ''' % settings.SITE.using_text()) #~ self.stream.write(settings.SITE.welcome_text()) self.stream.write(''' """ from __future__ import unicode_literals import logging logger = logging.getLogger('%s') import os ''' % __name__) if False: self.stream.write(""" os.environ['DJANGO_SETTINGS_MODULE'] = '%s' """ % settings.SETTINGS_MODULE) self.stream.write('SOURCE_VERSION = %r\n' % str(current_version)) self.stream.write(''' from decimal import Decimal from datetime import datetime as dt from datetime import time,date from django.conf import settings from django.contrib.contenttypes.models import ContentType from lino.utils.dpy import create_mti_child from lino.utils.dpy import DpyLoader from lino.core.utils import resolve_model def new_content_type_id(m): if m is None: return m ct = ContentType.objects.get_for_model(m) if ct is None: return None return ct.pk ''') s = ','.join([ '%s=values[%d]' % (lng.name, lng.index) for lng in settings.SITE.languages ]) self.stream.write(''' def bv2kw(fieldname, values): """ Needed if `Site.languages` changed between dumpdata and loaddata """ return settings.SITE.babelkw(fieldname, %s) ''' % s) self.models = [ m for m in sorted_models_list() if not issubclass(m, (ContentType, Session)) ] for model in self.models: self.stream.write( '%s = resolve_model("%s")\n' % (full_model_name(model, '_'), full_model_name(model))) self.stream.write('\n') self.models = self.sort_models(self.models) self.stream.write('\n') for model in self.models: fields = [ f for f, m in model._meta.get_fields_with_model() if m is None ] if AFTER17: fields = [f for f in fields if f.concrete] for f in fields: if getattr(f, 'auto_now_add', False): raise Exception( "%s.%s.auto_now_add is True : values will be lost!" % (full_model_name(model), f.name)) #~ fields = model._meta.local_fields #~ fields = [f for f in model._meta.fields if f.serialize] #~ fields = [f for f in model._meta.local_fields if f.serialize] self.stream.write('def create_%s(%s):\n' % (model._meta.db_table, ', '.join([ f.attname for f in fields if not getattr(f, '_lino_babel_field', False) ]))) if model._meta.parents: if len(model._meta.parents) != 1: msg = "%s : model._meta.parents is %r" % ( model, model._meta.parents) raise Exception(msg) pm, pf = model._meta.parents.items()[0] child_fields = [f for f in fields if f != pf] if child_fields: attrs = ',' + ','.join([ '%s=%s' % (f.attname, f.attname) for f in child_fields ]) else: attrs = '' #~ self.stream.write(' return insert_child(%s.objects.get(pk=%s),%s%s)\n' % ( #~ full_model_name(pm,'_'),pf.attname,full_model_name(model,'_'),attrs)) self.stream.write(' return create_mti_child(%s,%s,%s%s)\n' % (full_model_name(pm, '_'), pf.attname, full_model_name(model, '_'), attrs)) else: self.stream.write(" kw = dict()\n") for f in fields: if getattr(f, '_lino_babel_field', False): continue elif isinstance(f, (BabelCharField, BabelTextField)): self.stream.write( ' if %s is not None: kw.update(bv2kw(%r,%s))\n' % (f.attname, f.attname, f.attname)) else: if isinstance(f, models.DecimalField): self.stream.write( ' if %s is not None: %s = Decimal(%s)\n' % (f.attname, f.attname, f.attname)) elif isinstance( f, models.ForeignKey) and f.rel.to is ContentType: #~ self.stream.write( #~ ' %s = ContentType.objects.get_for_model(%s).pk\n' % ( #~ f.attname,f.attname)) self.stream.write( ' %s = new_content_type_id(%s)\n' % (f.attname, f.attname)) self.stream.write(' kw.update(%s=%s)\n' % (f.attname, f.attname)) self.stream.write(' return %s(**kw)\n\n' % full_model_name(model, '_')) self.stream.write('\n') #~ used_models = set() self.stream.write(""" def main(): loader = DpyLoader(globals()) from django.core.management import call_command # call_command('initdb', interactive=False) call_command('initdb') os.chdir(os.path.dirname(__file__)) loader.initialize() """) for model in progress.bar(self.models): filename = '%s.py' % model._meta.db_table filename = os.path.join(self.output_dir, filename) # puts("Writing {0}...".format(filename)) stream = file(filename, 'wt') stream.write('# -*- coding: UTF-8 -*-\n') qs = model.objects.all() try: stream.write( 'logger.info("Loading %d objects to table %s...")\n' % (qs.count(), model._meta.db_table)) fields = [ f for f, m in model._meta.get_fields_with_model() if m is None ] fields = [ f for f in fields if not getattr(f, '_lino_babel_field', False) ] if AFTER17: fields = [f for f in fields if f.concrete] stream.write("# fields: %s\n" % ', '.join([f.name for f in fields])) for obj in qs: self.count_objects += 1 #~ used_models.add(model) stream.write( 'loader.save(create_%s(%s))\n' % (obj._meta.db_table, ','.join( [self.value2string(obj, f) for f in fields]))) stream.write('\n') stream.write('loader.flush_deferred_objects()\n') except ProgrammingError as e: if not self.options['tolerate']: raise stream.write('\n') logger.warning("Tolerating database error %s in %s", e, model._meta.db_table) msg = ("The data of this table has not been dumped" "because an error {0} occured.").format(e) stream.write('raise Exception("{0}")\n'.format(msg)) stream.close() #~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % ) self.stream.write(' execfile("%s.py")\n' % model._meta.db_table) self.stream.write(' loader.finalize()\n') # self.stream.write( # ' logger.info("Loaded %d objects",loader.count_objects)\n') self.stream.write(""" if __name__ == '__main__': main() """) #~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n') self.stream.close()
def write_files(self): puts("Writing {0}...".format(self.main_file)) self.stream = open(self.main_file, 'wt') current_version = settings.SITE.version self.stream.write('''\ #!/usr/bin/env python # -*- coding: UTF-8 -*- # This is a Python dump created using dump2py. # DJANGO_SETTINGS_MODULE was %r, TIME_ZONE was %r. ''' % (settings.SETTINGS_MODULE, settings.TIME_ZONE)) self.stream.write(''' from __future__ import unicode_literals import logging logger = logging.getLogger('%s') ''' % __name__) self.stream.write('SOURCE_VERSION = %r\n' % str(current_version)) self.stream.write(''' import os import six from decimal import Decimal from datetime import datetime from datetime import time, date from django.conf import settings from django.utils.timezone import make_aware, utc from django.core.management import call_command # from django.contrib.contenttypes.models import ContentType from lino.utils.dpy import create_mti_child from lino.utils.dpy import DpyLoader from lino.core.utils import resolve_model if settings.USE_TZ: def dt(*args): return make_aware(datetime(*args), timezone=utc) else: def dt(*args): return datetime(*args) def new_content_type_id(m): if m is None: return m ct = settings.SITE.models.contenttypes.ContentType.objects.get_for_model(m) if ct is None: return None return ct.pk def pmem(): # Thanks to https://stackoverflow.com/questions/938733/total-memory-used-by-python-process process = psutil.Process(os.getpid()) print(process.memory_info().rss) def execfile(fn, *args): logger.info("Execute file %s ...", fn) six.exec_(compile(open(fn, "rb").read(), fn, 'exec'), *args) # pmem() # requires pip install psutil ''') s = ','.join([ '%s=values[%d]' % (lng.name, lng.index) for lng in settings.SITE.languages ]) self.stream.write(''' def bv2kw(fieldname, values): """ Needed if `Site.languages` changed between dumpdata and loaddata """ return settings.SITE.babelkw(fieldname, %s) ''' % s) self.models = sorted_models_list() if settings.SITE.is_installed('contenttypes'): from django.contrib.contenttypes.models import ContentType self.models = [ m for m in self.models if not issubclass(m, ContentType) ] if settings.SITE.is_installed('sessions'): from django.contrib.sessions.models import Session self.models = [ m for m in self.models if not issubclass(m, Session) ] for model in self.models: self.stream.write( '%s = resolve_model("%s")\n' % (full_model_name(model, '_'), full_model_name(model))) self.stream.write('\n') self.models = self.sort_models(self.models) self.stream.write('\n') for model in self.models: write_create_function(model, self.stream) self.stream.write('\n') #~ used_models = set() self.stream.write(""" def main(args): loader = DpyLoader(globals(), quick=args.quick) from django.core.management import call_command call_command('initdb', interactive=args.interactive) os.chdir(os.path.dirname(__file__)) loader.initialize() args = (globals(), locals()) """) max_row_count = self.options['max_row_count'] for model in progress.bar(self.models): try: qs = model.objects.all() total_count = qs.count() except DatabaseError as e: self.database_errors += 1 if not self.options['tolerate']: raise self.stream.write('\n') logger.warning("Tolerating database error %s in %s", e, model._meta.db_table) msg = ("The data of table {0} has not been dumped" "because an error {1} occured.").format( model._meta.db_table, e) self.stream.write('raise Exception("{0}")\n'.format(msg)) continue fields = [ f for f in model._meta.get_fields() if f.concrete and f.model is model ] fields = [ f for f in fields if not getattr(f, '_lino_babel_field', False) ] chunks = [] # list of tuples (i, filename, queryset) if total_count > max_row_count: num_files = (total_count // max_row_count) + 1 for i in range(num_files): o1 = max_row_count * i o2 = max_row_count * (i + 1) t = (i + 1, '%s_%d.py' % (model._meta.db_table, i + 1), qs[o1:o2]) chunks.append(t) else: chunks.append((1, '%s.py' % model._meta.db_table, qs)) for i, filename, qs in chunks: self.stream.write(' execfile("%s", *args)\n' % filename) filename = os.path.join(self.output_dir, filename) # puts("Writing {0}...".format(filename)) # stream = file(filename, 'wt') stream = open(filename, 'wt') stream.write('# -*- coding: UTF-8 -*-\n') txt = "%d objects" % total_count if len(chunks) > 1: txt += " (part %d of %d)" % (i, len(chunks)) stream.write('logger.info("Loading %s to table %s...")\n' % (txt, model._meta.db_table)) stream.write("# fields: %s\n" % ', '.join([f.name for f in fields])) for obj in qs: self.count_objects += 1 #~ used_models.add(model) stream.write( 'loader.save(create_%s(%s))\n' % (obj._meta.db_table, ','.join( [self.value2string(obj, f) for f in fields]))) stream.write('\n') if i == len(chunks): stream.write('loader.flush_deferred_objects()\n') stream.close() #~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % ) self.stream.write(' loader.finalize()\n') # 20180416 why was the following message commented out? # reactivated it because otherwise we have no log entry when # the process has finished. self.stream.write( ' logger.info("Loaded %d objects", loader.count_objects)\n') self.stream.write(" call_command('resetsequences')\n") self.stream.write(""" if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description='Restore the data.') parser.add_argument('--noinput', dest='interactive', action='store_false', default=True, help="Don't ask for confirmation before flushing the database.") parser.add_argument('--quick', dest='quick', action='store_true',default=False, help='Do not call full_clean() on restored instances.') args = parser.parse_args() main(args) """) #~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n') self.stream.close()
def write_files(self): puts("Writing {0}...".format(self.main_file)) self.stream = open(self.main_file, 'wt') current_version = settings.SITE.version self.stream.write('''\ #!/usr/bin/env python # -*- coding: UTF-8 -*- # This is a Python dump created using dump2py. # DJANGO_SETTINGS_MODULE was %r, TIME_ZONE was %r. ''' % (settings.SETTINGS_MODULE, settings.TIME_ZONE)) self.stream.write(''' from __future__ import unicode_literals import logging logger = logging.getLogger('%s') import os ''' % __name__) self.stream.write('SOURCE_VERSION = %r\n' % str(current_version)) self.stream.write(''' from decimal import Decimal from datetime import datetime from datetime import time, date from django.conf import settings from django.utils.timezone import make_aware # from django.contrib.contenttypes.models import ContentType from lino.utils.dpy import create_mti_child from lino.utils.dpy import DpyLoader from lino.core.utils import resolve_model if settings.USE_TZ: def dt(*args): return make_aware(datetime(*args)) else: def dt(*args): return datetime(*args) def new_content_type_id(m): if m is None: return m ct = settings.SITE.models.contenttypes.ContentType.objects.get_for_model(m) if ct is None: return None return ct.pk ''') s = ','.join([ '%s=values[%d]' % (lng.name, lng.index) for lng in settings.SITE.languages]) self.stream.write(''' def bv2kw(fieldname, values): """ Needed if `Site.languages` changed between dumpdata and loaddata """ return settings.SITE.babelkw(fieldname, %s) ''' % s) self.models = sorted_models_list() if settings.SITE.is_installed('contenttypes'): from django.contrib.contenttypes.models import ContentType self.models = [m for m in self.models if not issubclass(m, ContentType)] if settings.SITE.is_installed('sessions'): from django.contrib.sessions.models import Session self.models = [m for m in self.models if not issubclass(m, Session)] for model in self.models: self.stream.write('%s = resolve_model("%s")\n' % ( full_model_name(model, '_'), full_model_name(model))) self.stream.write('\n') self.models = self.sort_models(self.models) self.stream.write('\n') for model in self.models: fields = [f for f in model._meta.get_fields() if f.concrete and f.model is model] for f in fields: if getattr(f, 'auto_now_add', False): raise Exception("%s.%s.auto_now_add is True : values will be lost!" % ( full_model_name(model), f.name)) #~ fields = model._meta.local_fields #~ fields = [f for f in model._meta.fields if f.serialize] #~ fields = [f for f in model._meta.local_fields if f.serialize] self.stream.write('def create_%s(%s):\n' % ( model._meta.db_table, ', '.join([ f.attname for f in fields if not getattr(f, '_lino_babel_field', False)]))) for f in fields: if isinstance(f, models.DecimalField): self.stream.write( ' if %s is not None: %s = Decimal(%s)\n' % ( f.attname, f.attname, f.attname)) elif isinstance(f, ChoiceListField): lstname = 'settings.SITE.modules.{0}.{1}'.format( f.choicelist.app_label, f.choicelist.__name__) ln = ' if {0}: {0} = {1}.get_by_value({0})\n' self.stream.write(ln.format(f.attname, lstname)) elif is_pointer_to_contenttype(f): self.stream.write( ' %s = new_content_type_id(%s)\n' % ( f.attname, f.attname)) if model._meta.parents: if len(model._meta.parents) != 1: msg = "%s : model._meta.parents is %r" % ( model, model._meta.parents) raise Exception(msg) pm, pf = list(model._meta.parents.items())[0] child_fields = [f for f in fields if f != pf] if child_fields: attrs = ',' + ','.join([ '%s=%s' % (f.attname, f.attname) for f in child_fields]) else: attrs = '' self.stream.write( ' return create_mti_child(%s, %s, %s%s)\n' % ( full_model_name(pm, '_'), pf.attname, full_model_name(model, '_'), attrs)) else: self.stream.write(" kw = dict()\n") for f in fields: if getattr(f, '_lino_babel_field', False): continue elif isinstance(f, (BabelCharField, BabelTextField)): self.stream.write( ' if %s is not None: kw.update(bv2kw(%r,%s))\n' % ( f.attname, f.attname, f.attname)) else: self.stream.write( ' kw.update(%s=%s)\n' % (f.attname, f.attname)) self.stream.write(' return %s(**kw)\n\n' % full_model_name(model, '_')) self.stream.write('\n') #~ used_models = set() self.stream.write(""" def main(args): loader = DpyLoader(globals()) from django.core.management import call_command call_command('initdb', interactive=args.interactive) os.chdir(os.path.dirname(__file__)) loader.initialize() """) for model in progress.bar(self.models): filename = '%s.py' % model._meta.db_table filename = os.path.join(self.output_dir, filename) # puts("Writing {0}...".format(filename)) # stream = file(filename, 'wt') stream = open(filename, 'wt') stream.write('# -*- coding: UTF-8 -*-\n') qs = model.objects.all() try: stream.write( 'logger.info("Loading %d objects to table %s...")\n' % ( qs.count(), model._meta.db_table)) fields = [f for f in model._meta.get_fields() if f.concrete and f.model is model] fields = [ f for f in fields if not getattr(f, '_lino_babel_field', False)] stream.write( "# fields: %s\n" % ', '.join( [f.name for f in fields])) for obj in qs: self.count_objects += 1 #~ used_models.add(model) stream.write('loader.save(create_%s(%s))\n' % ( obj._meta.db_table, ','.join([self.value2string(obj, f) for f in fields]))) stream.write('\n') stream.write('loader.flush_deferred_objects()\n') except DatabaseError as e: self.database_errors += 1 if not self.options['tolerate']: raise stream.write('\n') logger.warning("Tolerating database error %s in %s", e, model._meta.db_table) msg = ("The data of this table has not been dumped" "because an error {0} occured.").format(e) stream.write('raise Exception("{0}")\n'.format(msg)) stream.close() #~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % ) self.stream.write(' execfile("%s.py")\n' % model._meta.db_table) self.stream.write( ' loader.finalize()\n') # self.stream.write( # ' logger.info("Loaded %d objects",loader.count_objects)\n') self.stream.write(""" if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description='Restore the data.') parser.add_argument('--noinput', dest='interactive', action='store_false', default=True, help="Don't ask for confirmation before flushing the database.") args = parser.parse_args() main(args) """) #~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n') self.stream.close()
def write_files(self): puts("Writing {0}...".format(self.main_file)) self.stream = open(self.main_file, 'wt') current_version = settings.SITE.version self.stream.write('''\ #!/usr/bin/env python # -*- coding: UTF-8 -*- # This is a Python dump created using dump2py. # DJANGO_SETTINGS_MODULE was %r, TIME_ZONE was %r. ''' % (settings.SETTINGS_MODULE, settings.TIME_ZONE)) self.stream.write(''' from __future__ import unicode_literals import logging logger = logging.getLogger('%s') ''' % __name__) self.stream.write('SOURCE_VERSION = %r\n' % str(current_version)) self.stream.write(''' import os import six from decimal import Decimal from datetime import datetime from datetime import time, date from django.conf import settings from django.utils.timezone import make_aware, utc from django.core.management import call_command # from django.contrib.contenttypes.models import ContentType from lino.utils.dpy import create_mti_child from lino.utils.dpy import DpyLoader from lino.core.utils import resolve_model if settings.USE_TZ: def dt(*args): return make_aware(datetime(*args), timezone=utc) else: def dt(*args): return datetime(*args) def new_content_type_id(m): if m is None: return m ct = settings.SITE.models.contenttypes.ContentType.objects.get_for_model(m) if ct is None: return None return ct.pk def pmem(): # Thanks to https://stackoverflow.com/questions/938733/total-memory-used-by-python-process process = psutil.Process(os.getpid()) print(process.memory_info().rss) def execfile(fn, *args): logger.info("Execute file %s ...", fn) six.exec_(compile(open(fn, "rb").read(), fn, 'exec'), *args) # pmem() # requires pip install psutil ''') s = ','.join([ '%s=values[%d]' % (lng.name, lng.index) for lng in settings.SITE.languages]) self.stream.write(''' def bv2kw(fieldname, values): """ Needed if `Site.languages` changed between dumpdata and loaddata """ return settings.SITE.babelkw(fieldname, %s) ''' % s) self.models = sorted_models_list() if settings.SITE.is_installed('contenttypes'): from django.contrib.contenttypes.models import ContentType self.models = [m for m in self.models if not issubclass(m, ContentType)] if settings.SITE.is_installed('sessions'): from django.contrib.sessions.models import Session self.models = [m for m in self.models if not issubclass(m, Session)] for model in self.models: self.stream.write('%s = resolve_model("%s")\n' % ( full_model_name(model, '_'), full_model_name(model))) self.stream.write('\n') self.models = self.sort_models(self.models) self.stream.write('\n') for model in self.models: write_create_function(model, self.stream) self.stream.write('\n') #~ used_models = set() self.stream.write(""" def main(args): loader = DpyLoader(globals(), quick=args.quick) from django.core.management import call_command call_command('initdb', interactive=args.interactive) os.chdir(os.path.dirname(__file__)) loader.initialize() args = (globals(), locals()) """) max_row_count = self.options['max_row_count'] for model in progress.bar(self.models): try: qs = model.objects.all() total_count = qs.count() except DatabaseError as e: self.database_errors += 1 if not self.options['tolerate']: raise self.stream.write('\n') logger.warning("Tolerating database error %s in %s", e, model._meta.db_table) msg = ("The data of table {0} has not been dumped" "because an error {1} occured.").format( model._meta.db_table, e) self.stream.write('raise Exception("{0}")\n'.format(msg)) continue fields = [f for f in model._meta.get_fields() if f.concrete and f.model is model] fields = [ f for f in fields if not getattr(f, '_lino_babel_field', False)] chunks = [] # list of tuples (i, filename, queryset) if total_count > max_row_count: num_files = (total_count // max_row_count) + 1 for i in range(num_files): o1 = max_row_count * i o2 = max_row_count * (i+1) t = (i+1, '%s_%d.py' % (model._meta.db_table, i+1), qs[o1:o2]) chunks.append(t) else: chunks.append((1, '%s.py' % model._meta.db_table, qs)) for i, filename, qs in chunks: self.stream.write(' execfile("%s", *args)\n' % filename) filename = os.path.join(self.output_dir, filename) # puts("Writing {0}...".format(filename)) # stream = file(filename, 'wt') stream = open(filename, 'wt') stream.write('# -*- coding: UTF-8 -*-\n') txt = "%d objects" % total_count if len(chunks) > 1: txt += " (part %d of %d)" % (i, len(chunks)) stream.write( 'logger.info("Loading %s to table %s...")\n' % ( txt, model._meta.db_table)) stream.write( "# fields: %s\n" % ', '.join( [f.name for f in fields])) for obj in qs: self.count_objects += 1 #~ used_models.add(model) stream.write('loader.save(create_%s(%s))\n' % ( obj._meta.db_table, ','.join([self.value2string(obj, f) for f in fields]))) stream.write('\n') if i == len(chunks): stream.write('loader.flush_deferred_objects()\n') stream.close() #~ self.stream.write('\nfilename = os.path.join(os.path.dirname(__file__),"%s.py")\n' % ) self.stream.write( ' loader.finalize()\n') # 20180416 why was the following message commented out? # reactivated it because otherwise we have no log entry when # the process has finished. self.stream.write( ' logger.info("Loaded %d objects", loader.count_objects)\n') self.stream.write( " call_command('resetsequences')\n") self.stream.write(""" if __name__ == '__main__': import argparse parser = argparse.ArgumentParser(description='Restore the data.') parser.add_argument('--noinput', dest='interactive', action='store_false', default=True, help="Don't ask for confirmation before flushing the database.") parser.add_argument('--quick', dest='quick', action='store_true',default=False, help='Do not call full_clean() on restored instances.') args = parser.parse_args() main(args) """) #~ self.stream.write('\nsettings.SITE.load_from_file(globals())\n') self.stream.close()