Esempio n. 1
0
def main():

    parser = optparse.OptionParser()
    parser.set_defaults(action="none",
                        dhost="localhost",
                        dport="5432",
                        ddriver="pgsql",
                        duser="******",
                        dpasswd="",
                        loaddir=".",
                        quiet=False,
                        verbose=False,
                        debug=False,
                        full=False,
                        diskcopy=False,
                        getdiskcopy=False,
                        rebuildtables=False,
                        flscriptparser=False,
                        addchecks=False,
                        preparse=False,
                        rebuildalone=False,
                        reindex=False,
                        transactions=False,
                        files_loaded=[],
                        seqsync=None,
                        pgsqlnodrop=False,
                        forgottables=False,
                        cleanfiles=False,
                        nopg=False,
                        updatemodules=False,
                        modules={})
    parser.add_option("--onlyalone",
                      help="Forbid rebuilds if other users are connected",
                      dest="rebuildalone",
                      action="store_true")

    parser.add_option(
        "--add-pkeys",
        help="Try to add new primary keys if they're not on the dest database",
        dest="addpkeys",
        action="store_true")

    parser.add_option(
        "--index",
        help=
        "Drop indexes and create them again whenever the MTD file is processed",
        dest="index",
        action="store_true")

    parser.add_option(
        "--reindex",
        help=
        "Drop indexes and create them again whenever the MTD file is processed",
        dest="reindex",
        action="store_true")

    parser.add_option(
        "--loindex",
        help="Disables most indexes used by Eneboo to enhance write speed",
        dest="loindex",
        action="store_true")

    parser.add_option("--diskcopy",
                      help="Create a backup .pydbabackup",
                      dest="diskcopy",
                      action="store_true")

    parser.add_option("--getdiskcopy",
                      help="Loads a .pydbabackup",
                      dest="getdiskcopy")

    parser.add_option(
        "--addchecks",
        help="Creates database checks (constraints, unique indexes)",
        dest="addchecks",
        action="store_true")

    parser.add_option("--preparse",
                      help="Checks and parses all MTD's",
                      dest="preparse",
                      action="store_true")

    parser.add_option("--transactional",
                      help="Use transactions inside PyDBA",
                      dest="transactions",
                      action="store_true")

    parser.add_option("--cleanfiles",
                      help="Purge old data on flfiles table",
                      dest="cleanfiles",
                      action="store_true")

    parser.add_option("--debug",
                      help="Tons of debug output",
                      dest="debug",
                      action="store_true")

    parser.add_option(
        "--safe",
        help="Enable safe mode. Disables table rebuild and psql load/drop.",
        dest="safe",
        action="store_true")

    parser.add_option("--pg-nodrop",
                      help="Disables dropping psql objects before load.",
                      dest="pgsqlnodrop",
                      action="store_true")

    parser.add_option("--nopg",
                      help="Disables creation/deletion of objects",
                      dest="nopg",
                      action="store_true")

    parser.add_option("-v",
                      help="Be more verbose",
                      dest="verbose",
                      action="store_true")

    parser.add_option("-q",
                      help="Quiet. Produce less output.",
                      dest="quiet",
                      action="store_true")

    parser.add_option("-P",
                      help="Parse QS Files using flscriptarser",
                      dest="flscriptparser",
                      action="store_true")

    parser.add_option("--full",
                      help="Deeper and slower checks",
                      dest="full",
                      action="store_true")

    parser.add_option("--rebuildtables",
                      help="DROP and CREATE tables again",
                      dest="rebuildtables",
                      action="store_true")

    parser.add_option(
        "--forgottables",
        help=
        "Update Metadata always and forgot what tables are pending a rebuild",
        dest="forgottables",
        action="store_true")

    parser.add_option("--updatemodules",
                      help="Update Modules doing a DELETE - INSERT",
                      dest="updatemodules",
                      action="store_true")

    g_action = optparse.OptionGroup(parser, "Actions",
                                    "You MUST provide one of :")

    # ******************* ACTIONS
    g_action.add_option("-T",
                        "--TEST",
                        action="store_const",
                        const="test_pydba",
                        dest="action",
                        help="Do some checks on PyDBA")

    g_action.add_option("-l",
                        "--lmod",
                        action="store_const",
                        const="load_module",
                        dest="action",
                        help="load modules")

    g_action.add_option("-L",
                        "--Lmod",
                        action="store_const",
                        const="load_module_sql",
                        dest="action",
                        help="load create SQL and modules")

    g_action.add_option("-r",
                        "--run",
                        action="store_const",
                        const="run_module",
                        dest="action",
                        help="load modules and run Eneboo")

    g_action.add_option("-O",
                        "--olap",
                        action="store_const",
                        const="setup_olap",
                        dest="action",
                        help="setup olap tables (obsolete, use -Y)")

    g_action.add_option("-Y",
                        "--export-yaml",
                        action="store_const",
                        const="setup_olap",
                        dest="action",
                        help="export metadata as Yaml")

    g_action.add_option("-m",
                        "--reload-mtd",
                        action="store_const",
                        const="reload_mtd",
                        dest="action",
                        help="Parses MTD files on DB and complete tables")

    g_action.add_option("-R",
                        "--repairdb",
                        action="store_const",
                        const="repair_db",
                        dest="action",
                        help="Execute tests to repair DB")

    g_action.add_option("-D",
                        "--dump",
                        action="store_const",
                        const="dump_db",
                        dest="action",
                        help="Dumps flfiles to disk")

    g_action.add_option("--create",
                        "--createdb",
                        action="store_const",
                        const="create_db",
                        dest="action",
                        help="Create a new Database with basic fl* tables")

    g_action.add_option("-c",
                        "--check",
                        action="store_const",
                        const="check",
                        dest="action",
                        help="Check relations for the DB")

    g_action.add_option("-M",
                        "--mysql2pgsql",
                        action="store_const",
                        const="mysql_convert",
                        dest="action",
                        help="Convert MySQL Database to PostgreSQL")

    parser.add_option_group(g_action)
    # ******************* CONFIG

    g_options = optparse.OptionGroup(
        parser, "Options",
        "Optional database and host selection. Some actions use them")
    g_options.add_option("--dhost",
                         dest="dhost",
                         help="Set the destination host")
    g_options.add_option("--dport",
                         dest="dport",
                         help="Set the destination port")
    g_options.add_option("--ddriver",
                         dest="ddriver",
                         help="Set the driver for dest DB (mysql; pgsql)")
    g_options.add_option("--ddb",
                         dest="ddb",
                         help="Set DDB as destination Database")
    g_options.add_option("--duser",
                         dest="duser",
                         help="Provide user for DB connection")
    g_options.add_option("--dpasswd",
                         dest="dpasswd",
                         help="Provide password for DB connection")

    g_options.add_option("--ohost", dest="ohost", help="Set the origin host")
    g_options.add_option("--oport", dest="oport", help="Set the origin port")
    g_options.add_option("--odriver",
                         dest="odriver",
                         help="Set the driver for origin DB (mysql; pgsql)")
    g_options.add_option("--odb",
                         dest="odb",
                         help="Set DDB as origin Database")
    g_options.add_option("--ouser",
                         dest="ouser",
                         help="Provide user for origin DB connection")
    g_options.add_option("--opasswd",
                         dest="opasswd",
                         help="Provide password for origin DB connection")

    g_options.add_option("--loaddir",
                         dest="loaddir",
                         help="Select Working Directory for Modules")

    g_options.add_option(
        "--loadbaselec",
        dest="loadbaselec",
        help="Import CSV File to Baselec table ('-' for stdin)")

    g_options.add_option("--loadini",
                         dest="loadini",
                         help="load and execute INI file")

    parser.add_option_group(g_options)

    #parser.add_option("-f", "--file", dest="filename",
    #                  help="write report to FILE", metavar="FILE")
    #parser.add_option("-q", "--quiet",
    #                  action="store_false", dest="verbose", default=True,
    #                  help="don't print status messages to stdout")

    for param in os.environ.keys():
        if param[:5] == "PYDBA":
            value = os.environ[param]
            if param == "PYDBA_DHOST":
                parser.set_defaults(dhost=value)
            elif param == "PYDBA_DUSER":
                parser.set_defaults(duser=value)
            elif param == "PYDBA_DPASSWD":
                parser.set_defaults(dpasswd=value)
            elif param == "PYDBA_DPORT":
                parser.set_defaults(dport=value)
            else:
                print "Unknown env var: %20s %s" % (param, value)

    (options, args) = parser.parse_args()

    if options.loadini:
        exec_ini(options, options.loadini)

    if options.ddriver and not options.odriver:
        options.odriver = options.ddriver
    if options.dhost and not options.ohost: options.ohost = options.dhost
    if options.duser and not options.ouser: options.ouser = options.duser
    if options.dpasswd and not options.opasswd:
        options.opasswd = options.dpasswd
    if options.ddb and not options.odb: options.odb = options.ddb
    if options.dport and not options.oport: options.oport = options.dport
    if options.diskcopy:
        options.full = True
        #options.rebuildtables = True

    if options.safe:
        if options.rebuildtables:
            print u"WARN: Se solicitó rebuildtables, pero safemode está activo."
            options.rebuildtables = False

    if (options.action == "setup_olap" or options.action == "check"
            or options.getdiskcopy or options.diskcopy):
        options.preparse = True

    if (options.action == "none"):
        print "You must provide at least one action"

    elif (options.action == "run_module"):
        db = load_module(options, preparse=options.preparse)
        repair_db(options, db)
        try:
            subprocess.call([
                "eneboo", "-silentconn", ":".join([
                    options.ddb, options.duser, "PostgreSQL", options.dhost,
                    options.dport, options.dpasswd
                ])
            ])
        except KeyboardInterrupt, e:
            print "<KeyboardInterrupt>"
Esempio n. 2
0
def main():

    parser = optparse.OptionParser()
    parser.set_defaults(
        action="none",
        dhost="localhost",
        dport="5432",
        ddriver="pgsql",
        duser="******",
        dpasswd="",
        loaddir=".",
        quiet=False,
        verbose=False,
        debug=False,
        full=False,
        diskcopy=False,
        getdiskcopy=False,
        rebuildtables=False,
        flscriptparser=False,
        addchecks=False,
        preparse=False,
        rebuildalone=False,
        reindex=False,
        transactions=False,
        files_loaded=[],
        seqsync=None,
        pgsqlnodrop=False,
        forgottables=False,
        cleanfiles=False,
        nopg=False,
        updatemodules=False,
        modules={},
    )
    parser.add_option(
        "--onlyalone", help="Forbid rebuilds if other users are connected", dest="rebuildalone", action="store_true"
    )

    parser.add_option(
        "--add-pkeys",
        help="Try to add new primary keys if they're not on the dest database",
        dest="addpkeys",
        action="store_true",
    )

    parser.add_option(
        "--index",
        help="Drop indexes and create them again whenever the MTD file is processed",
        dest="index",
        action="store_true",
    )

    parser.add_option(
        "--reindex",
        help="Drop indexes and create them again whenever the MTD file is processed",
        dest="reindex",
        action="store_true",
    )

    parser.add_option(
        "--loindex",
        help="Disables most indexes used by Eneboo to enhance write speed",
        dest="loindex",
        action="store_true",
    )

    parser.add_option("--diskcopy", help="Create a backup .pydbabackup", dest="diskcopy", action="store_true")

    parser.add_option("--getdiskcopy", help="Loads a .pydbabackup", dest="getdiskcopy")

    parser.add_option(
        "--addchecks",
        help="Creates database checks (constraints, unique indexes)",
        dest="addchecks",
        action="store_true",
    )

    parser.add_option("--preparse", help="Checks and parses all MTD's", dest="preparse", action="store_true")

    parser.add_option("--transactional", help="Use transactions inside PyDBA", dest="transactions", action="store_true")

    parser.add_option("--cleanfiles", help="Purge old data on flfiles table", dest="cleanfiles", action="store_true")

    parser.add_option("--debug", help="Tons of debug output", dest="debug", action="store_true")

    parser.add_option(
        "--safe", help="Enable safe mode. Disables table rebuild and psql load/drop.", dest="safe", action="store_true"
    )

    parser.add_option(
        "--pg-nodrop", help="Disables dropping psql objects before load.", dest="pgsqlnodrop", action="store_true"
    )

    parser.add_option("--nopg", help="Disables creation/deletion of objects", dest="nopg", action="store_true")

    parser.add_option("-v", help="Be more verbose", dest="verbose", action="store_true")

    parser.add_option("-q", help="Quiet. Produce less output.", dest="quiet", action="store_true")

    parser.add_option("-P", help="Parse QS Files using flscriptarser", dest="flscriptparser", action="store_true")

    parser.add_option("--full", help="Deeper and slower checks", dest="full", action="store_true")

    parser.add_option("--rebuildtables", help="DROP and CREATE tables again", dest="rebuildtables", action="store_true")

    parser.add_option(
        "--forgottables",
        help="Update Metadata always and forgot what tables are pending a rebuild",
        dest="forgottables",
        action="store_true",
    )

    parser.add_option(
        "--updatemodules", help="Update Modules doing a DELETE - INSERT", dest="updatemodules", action="store_true"
    )

    g_action = optparse.OptionGroup(parser, "Actions", "You MUST provide one of :")

    # ******************* ACTIONS
    g_action.add_option(
        "-T", "--TEST", action="store_const", const="test_pydba", dest="action", help="Do some checks on PyDBA"
    )

    g_action.add_option("-l", "--lmod", action="store_const", const="load_module", dest="action", help="load modules")

    g_action.add_option(
        "-L", "--Lmod", action="store_const", const="load_module_sql", dest="action", help="load create SQL and modules"
    )

    g_action.add_option(
        "-r", "--run", action="store_const", const="run_module", dest="action", help="load modules and run Eneboo"
    )

    g_action.add_option(
        "-O",
        "--olap",
        action="store_const",
        const="setup_olap",
        dest="action",
        help="setup olap tables (obsolete, use -Y)",
    )

    g_action.add_option(
        "-Y", "--export-yaml", action="store_const", const="setup_olap", dest="action", help="export metadata as Yaml"
    )

    g_action.add_option(
        "-m",
        "--reload-mtd",
        action="store_const",
        const="reload_mtd",
        dest="action",
        help="Parses MTD files on DB and complete tables",
    )

    g_action.add_option(
        "-R", "--repairdb", action="store_const", const="repair_db", dest="action", help="Execute tests to repair DB"
    )

    g_action.add_option(
        "-D", "--dump", action="store_const", const="dump_db", dest="action", help="Dumps flfiles to disk"
    )

    g_action.add_option(
        "--create",
        "--createdb",
        action="store_const",
        const="create_db",
        dest="action",
        help="Create a new Database with basic fl* tables",
    )

    g_action.add_option(
        "-c", "--check", action="store_const", const="check", dest="action", help="Check relations for the DB"
    )

    g_action.add_option(
        "-M",
        "--mysql2pgsql",
        action="store_const",
        const="mysql_convert",
        dest="action",
        help="Convert MySQL Database to PostgreSQL",
    )

    parser.add_option_group(g_action)
    # ******************* CONFIG

    g_options = optparse.OptionGroup(parser, "Options", "Optional database and host selection. Some actions use them")
    g_options.add_option("--dhost", dest="dhost", help="Set the destination host")
    g_options.add_option("--dport", dest="dport", help="Set the destination port")
    g_options.add_option("--ddriver", dest="ddriver", help="Set the driver for dest DB (mysql; pgsql)")
    g_options.add_option("--ddb", dest="ddb", help="Set DDB as destination Database")
    g_options.add_option("--duser", dest="duser", help="Provide user for DB connection")
    g_options.add_option("--dpasswd", dest="dpasswd", help="Provide password for DB connection")

    g_options.add_option("--ohost", dest="ohost", help="Set the origin host")
    g_options.add_option("--oport", dest="oport", help="Set the origin port")
    g_options.add_option("--odriver", dest="odriver", help="Set the driver for origin DB (mysql; pgsql)")
    g_options.add_option("--odb", dest="odb", help="Set DDB as origin Database")
    g_options.add_option("--ouser", dest="ouser", help="Provide user for origin DB connection")
    g_options.add_option("--opasswd", dest="opasswd", help="Provide password for origin DB connection")

    g_options.add_option("--loaddir", dest="loaddir", help="Select Working Directory for Modules")

    g_options.add_option("--loadbaselec", dest="loadbaselec", help="Import CSV File to Baselec table ('-' for stdin)")

    g_options.add_option("--loadini", dest="loadini", help="load and execute INI file")

    parser.add_option_group(g_options)

    # parser.add_option("-f", "--file", dest="filename",
    #                  help="write report to FILE", metavar="FILE")
    # parser.add_option("-q", "--quiet",
    #                  action="store_false", dest="verbose", default=True,
    #                  help="don't print status messages to stdout")

    for param in os.environ.keys():
        if param[:5] == "PYDBA":
            value = os.environ[param]
            if param == "PYDBA_DHOST":
                parser.set_defaults(dhost=value)
            elif param == "PYDBA_DUSER":
                parser.set_defaults(duser=value)
            elif param == "PYDBA_DPASSWD":
                parser.set_defaults(dpasswd=value)
            elif param == "PYDBA_DPORT":
                parser.set_defaults(dport=value)
            else:
                print "Unknown env var: %20s %s" % (param, value)

    (options, args) = parser.parse_args()

    if options.loadini:
        exec_ini(options, options.loadini)

    if options.ddriver and not options.odriver:
        options.odriver = options.ddriver
    if options.dhost and not options.ohost:
        options.ohost = options.dhost
    if options.duser and not options.ouser:
        options.ouser = options.duser
    if options.dpasswd and not options.opasswd:
        options.opasswd = options.dpasswd
    if options.ddb and not options.odb:
        options.odb = options.ddb
    if options.dport and not options.oport:
        options.oport = options.dport
    if options.diskcopy:
        options.full = True
        # options.rebuildtables = True

    if options.safe:
        if options.rebuildtables:
            print u"WARN: Se solicitó rebuildtables, pero safemode está activo."
            options.rebuildtables = False

    if options.action == "setup_olap" or options.action == "check" or options.getdiskcopy or options.diskcopy:
        options.preparse = True

    if options.action == "none":
        print "You must provide at least one action"

    elif options.action == "run_module":
        db = load_module(options, preparse=options.preparse)
        repair_db(options, db)
        try:
            subprocess.call(
                [
                    "eneboo",
                    "-silentconn",
                    ":".join([options.ddb, options.duser, "PostgreSQL", options.dhost, options.dport, options.dpasswd]),
                ]
            )
        except KeyboardInterrupt, e:
            print "<KeyboardInterrupt>"
Esempio n. 3
0
    elif (options.action == "run_module"):
        db = load_module(options, preparse=options.preparse)
        repair_db(options, db)
        try:
            subprocess.call([
                "eneboo", "-silentconn", ":".join([
                    options.ddb, options.duser, "PostgreSQL", options.dhost,
                    options.dport, options.dpasswd
                ])
            ])
        except KeyboardInterrupt, e:
            print "<KeyboardInterrupt>"

    elif (options.action == "load_module"):
        db = load_module(options, preparse=options.preparse)
        repair_db(options, db)
    elif (options.action == "load_module_sql"):
        db = create_sql(options)
        load_module(options, db, preparse=options.preparse)
        repair_db(options, db)
    elif (options.action == "setup_olap"):
        db = load_module(options, preparse=options.preparse)
        olap = procesarOLAP(db)
        olapfilename = "relationdata.yaml"
        f1 = open(olapfilename, "w")
        f1.write(olap)
        f1.close()
        print "Fichero %s guardado." % olapfilename

    elif (options.action == "check"):
Esempio n. 4
0
    elif options.action == "run_module":
        db = load_module(options, preparse=options.preparse)
        repair_db(options, db)
        try:
            subprocess.call(
                [
                    "eneboo",
                    "-silentconn",
                    ":".join([options.ddb, options.duser, "PostgreSQL", options.dhost, options.dport, options.dpasswd]),
                ]
            )
        except KeyboardInterrupt, e:
            print "<KeyboardInterrupt>"

    elif options.action == "load_module":
        db = load_module(options, preparse=options.preparse)
        repair_db(options, db)
    elif options.action == "load_module_sql":
        db = create_sql(options)
        load_module(options, db, preparse=options.preparse)
        repair_db(options, db)
    elif options.action == "setup_olap":
        db = load_module(options, preparse=options.preparse)
        olap = procesarOLAP(db)
        olapfilename = "relationdata.yaml"
        f1 = open(olapfilename, "w")
        f1.write(olap)
        f1.close()
        print "Fichero %s guardado." % olapfilename

    elif options.action == "check":