コード例 #1
0
    def handle(self, options, global_options, *args):
        from uliweb import orm
        from zipfile import ZipFile
        import shutil

        if args:
            message = """This command will delete all data of [%s]-[%s] before loading, 
are you sure to load data""" % (options.engine, ','.join(args))
        else:
            print "Failed! You should pass one or more tables name."
            sys.exit(1)

        ans = 'Y' if global_options.yes else get_answer(message, quit='q')

        # extract zip file to path
        if options.zipfile:
            if options.dir and not os.path.exists(options.dir):
                os.makedirs(options.dir)
            path = get_temppath(prefix='dump', dir=options.dir)
            if global_options.verbose:
                print "Extract path is %s" % path
            zipfile = None
            try:
                zipfile = ZipFile(options.zipfile, 'r')
                zipfile.extractall(path)
            except:
                log.exception(
                    "There are something wrong when extract zip file [%s]" %
                    options.zipfile)
                sys.exit(1)
            finally:
                if zipfile:
                    zipfile.close()
        else:
            path = os.path.join(options.dir, options.engine)
            if not os.path.exists(path):
                os.makedirs(path)

        engine = get_engine(options, global_options)

        tables = get_sorted_tables(
            get_tables(global_options.apps_dir,
                       engine_name=options.engine,
                       settings_file=global_options.settings,
                       tables=args,
                       local_settings_file=global_options.local_settings))
        _len = len(tables)

        for i, (name, t) in enumerate(tables):
            if t.__mapping_only__:
                if global_options.verbose:
                    msg = 'SKIPPED(Mapping Table)'
                    print '[%s] Loading %s...%s' % (
                        options.engine, show_table(name, t, i, _len), msg)
                continue
            if global_options.verbose:
                print '[%s] Loading %s...' % (options.engine,
                                              show_table(name, t, i, _len)),
            try:
                orm.Begin()
                filename = os.path.join(path, name + '.txt')
                if options.text:
                    format = 'txt'
                else:
                    format = None
                t = load_table(t,
                               filename,
                               engine,
                               delimiter=options.delimiter,
                               format=format,
                               encoding=options.encoding,
                               delete=ans == 'Y',
                               bulk=int(options.bulk),
                               engine_name=engine.engine_name)
                orm.Commit()
                if global_options.verbose:
                    print t
            except:
                log.exception(
                    "There are something wrong when loading table [%s]" % name)
                orm.Rollback()

        if options.zipfile:
            shutil.rmtree(path)
コード例 #2
0
    def handle(self, options, global_options, *args):
        from uliweb import orm
        from zipfile import ZipFile
        import shutil

        if args:
            message = """This command will delete all data of [%s]-[%s] before loading,
are you sure to load data""" % (options.engine, ','.join(args))
        else:
            message = """This command will delete whole database [%s] before loading,
are you sure to load data""" % options.engine

        ans = 'Y' if global_options.yes else get_answer(message)

        if ans != 'Y':
            return

        # extract zip file to path
        if options.zipfile:
            if options.dir and not os.path.exists(options.dir):
                os.makedirs(options.dir)
            path = get_temppath(prefix='dump', dir=options.dir)
            if global_options.verbose:
                print("Extract path is %s" % path)
            zipfile = None
            try:
                zipfile = ZipFile(options.zipfile, 'r')
                zipfile.extractall(path)
            except:
                log.exception("There are something wrong when extract zip file [%s]" % options.zipfile)
                sys.exit(1)
            finally:
                if zipfile:
                    zipfile.close()
        else:
            path = os.path.join(options.dir, options.engine)
            if not os.path.exists(path):
                os.makedirs(path)

        engine = get_engine(options, global_options)

        tables = get_sorted_tables(get_tables(global_options.apps_dir, args,
            engine_name=options.engine,
            settings_file=global_options.settings,
            local_settings_file=global_options.local_settings, all=options.all))
        _len = len(tables)

        def _f(table, filename, msg):
            session = orm.get_session()
            if session:
                session.close()
            orm.Begin()
            try:
                result = load_table(table, filename, engine, delimiter=options.delimiter,
                    format=format, encoding=options.encoding, delete=ans=='Y',
                    bulk=int(options.bulk), engine_name=engine.engine_name)
                if global_options.verbose:
                    print(msg, result)
                orm.Commit()
            except:
                import traceback
                traceback.print_exc()
                orm.Rollback()

        for i, (name, t) in enumerate(tables):
            if hasattr(t, '__mapping_only__') and t.__mapping_only__:
                if global_options.verbose:
                    msg = 'SKIPPED(Mapping Table)'
                    print('[%s] Loading %s...%s' % (options.engine, show_table(name, t, i, _len), msg))
                continue
            msg = ''
            if global_options.verbose:
                msg = '[%s] Loading %s...' % (options.engine, show_table(name, t, i, _len))
            try:
                orm.Begin()
                filename = os.path.join(path, name+'.txt')
                if options.text:
                    format = 'txt'
                else:
                    format = None

                    #fork process to run
                    if sys.platform != 'win32' and options.multi>1:
                        load_table_file(t, filename, options.multi, bulk=options.bulk)
                    else:
                        _f(t, filename, msg)
            except:
                log.exception("There are something wrong when loading table [%s]" % name)
                orm.Rollback()

        if options.zipfile:
            shutil.rmtree(path)
コード例 #3
0
ファイル: commands.py プロジェクト: wangaicc/uliweb
    def handle(self, options, global_options, *args):
        from uliweb import orm
        from zipfile import ZipFile
        import shutil

        if args:
            message = """This command will delete all data of [%s]-[%s] before loading, 
are you sure to load data""" % (options.engine, ','.join(args))
        else:
            print "Failed! You should pass one or more tables name."
            sys.exit(1)

        ans = 'Y' if global_options.yes else get_answer(message, quit='q')

        # extract zip file to path
        if options.zipfile:
            if options.dir and not os.path.exists(options.dir):
                os.makedirs(options.dir)
            path = get_temppath(prefix='dump', dir=options.dir)
            if global_options.verbose:
                print "Extract path is %s" % path
            zipfile = None
            try:
                zipfile = ZipFile(options.zipfile, 'r')
                zipfile.extractall(path)
            except:
                log.exception("There are something wrong when extract zip file [%s]" % options.zipfile)
                sys.exit(1)
            finally:
                if zipfile:
                    zipfile.close()
        else:
            path = os.path.join(options.dir, options.engine)
            if not os.path.exists(path):
                os.makedirs(path)


        engine = get_engine(options, global_options)

        tables = get_sorted_tables(get_tables(global_options.apps_dir, 
            engine_name=options.engine, 
            settings_file=global_options.settings, tables=args,
            local_settings_file=global_options.local_settings))
        _len = len(tables)

        def _f(table, filename, msg):
            session = orm.get_session()
            if session:
                session.close()
            orm.Begin()
            try:
                result = load_table(table, filename, engine, delimiter=options.delimiter,
                    format=format, encoding=options.encoding, delete=ans=='Y',
                    bulk=int(options.bulk), engine_name=engine.engine_name)
                if global_options.verbose:
                    print msg, result
                orm.Commit()
            except:
                orm.Rollback()

        manager = ProcessManager(options.multi)

        for i, (name, t) in enumerate(tables):
            if t.__mapping_only__:
                if global_options.verbose:
                    msg = 'SKIPPED(Mapping Table)'
                    print '[%s] Loading %s...%s' % (options.engine, show_table(name, t, i, _len), msg)
                continue
            if global_options.verbose:
                msg = '[%s] Loading %s...' % (options.engine, show_table(name, t, i, _len))
            try:
                filename = os.path.join(path, name+'.txt')
                if options.text:
                    format = 'txt'
                else:
                    format = None

                    #fork process to run
                    if sys.platform != 'win32' and options.multi>1:
                        manager(_f, t, filename, msg)
                    else:
                        _f(t, filename, msg)
            except:
                log.exception("There are something wrong when loading table [%s]" % name)

        manager.join()

        if options.zipfile:
            shutil.rmtree(path)
コード例 #4
0
ファイル: commands.py プロジェクト: hb44/uliweb
    def handle(self, options, global_options, *args):
        from uliweb import orm
        from zipfile import ZipFile
        import shutil

        if args:
            message = """This command will delete all data of [%s]-[%s] before loading, 
are you sure to load data""" % (options.engine, ','.join(args))
        else:
            message = """This command will delete whole database [%s] before loading, 
are you sure to load data""" % options.engine

        ans = 'Y' if global_options.yes else get_answer(message)

        if ans != 'Y':
            return

        # extract zip file to path
        if options.zipfile:
            if options.dir and not os.path.exists(options.dir):
                os.makedirs(options.dir)
            path = get_temppath(prefix='dump', dir=options.dir)
            if global_options.verbose:
                print "Extract path is %s" % path
            zipfile = None
            try:
                zipfile = ZipFile(options.zipfile, 'r')
                zipfile.extractall(path)
            except:
                log.exception("There are something wrong when extract zip file [%s]" % options.zipfile)
                sys.exit(1)
            finally:
                if zipfile:
                    zipfile.close()
        else:
            path = os.path.join(options.dir, options.engine)
            if not os.path.exists(path):
                os.makedirs(path)

        engine = get_engine(options, global_options)

        tables = get_sorted_tables(get_tables(global_options.apps_dir, args, 
            engine_name=options.engine, 
            settings_file=global_options.settings, 
            local_settings_file=global_options.local_settings, all=options.all))
        _len = len(tables)
        for i, (name, t) in enumerate(tables):
            if hasattr(t, '__mapping_only__') and t.__mapping_only__:
                if global_options.verbose:
                    msg = 'SKIPPED(Mapping Table)'
                    print '[%s] Loading %s...%s' % (options.engine, show_table(name, t, i, _len), msg)
                continue
            if global_options.verbose:
                print '[%s] Loading %s...' % (options.engine, show_table(name, t, i, _len)),
            try:
                orm.Begin()
                filename = os.path.join(path, name+'.txt')
                if options.text:
                    format = 'txt'
                else:
                    format = None
                t = load_table(t, filename, engine, delimiter=options.delimiter, 
                    format=format, encoding=options.encoding,
                    bulk=int(options.bulk), engine_name=engine.engine_name)
                orm.Commit()
                if global_options.verbose:
                    print t
                
            except:
                log.exception("There are something wrong when loading table [%s]" % name)
                orm.Rollback()

        if options.zipfile:
            shutil.rmtree(path)