Example #1
0
def create_shelf_from_csv():
    """
    Function to be called from the command line to convert a CSV based assignments file to a shelf.

    :return: exit code
    """
    import argparse
    import sys
    from dhcpkit.ipv6.option_handlers.csv import CSVBasedFixedAssignmentOptionHandler

    # Handle command line arguments
    parser = argparse.ArgumentParser(
        description="Assignments CSV to Shelf converter",
    )

    parser.add_argument("source", help="the source CSV file")
    parser.add_argument("destination", help="the destination shelf file")
    parser.add_argument("-v", "--verbosity", action="count", default=0, help="increase output verbosity")

    args = parser.parse_args()

    # Our logger is the root logger now
    global logger
    logger = logging.getLogger()

    # Don't filter on level in the root logger
    logger.setLevel(logging.NOTSET)

    # Output to sys.stdout
    stdout_handler = logging.StreamHandler(stream=sys.stdout)

    # Set level according to verbosity
    if args.verbosity >= 3:
        stdout_handler.setLevel(logging.DEBUG)
    elif args.verbosity == 2:
        stdout_handler.setLevel(logging.INFO)
    elif args.verbosity >= 1:
        stdout_handler.setLevel(logging.WARNING)
    else:
        stdout_handler.setLevel(logging.CRITICAL)

    logger.addHandler(stdout_handler)

    logger.info("Reading assignments from CSV file {}".format(args.source))
    assignments = CSVBasedFixedAssignmentOptionHandler.parse_csv_file(args.source)

    logger.info("Writing assignments to shelf file {}".format(args.destination))
    with shelve.open(args.destination, 'n') as shelf:
        for key, value in assignments:
            shelf[key] = value

        logger.info("Wrote {} assignments".format(len(shelf)))
Example #2
0
def create_sqlite_from_csv():
    """
    Function to be called from the command line to convert a CSV based assignments file to a sqlite database.

    :return: exit code
    """
    import argparse
    import sys
    from dhcpkit.ipv6.option_handlers.csv import CSVBasedFixedAssignmentOptionHandler

    # Handle command line arguments
    parser = argparse.ArgumentParser(description="Assignments CSV to SQLite converter")

    parser.add_argument("source", help="the source CSV file")
    parser.add_argument("destination", help="the destination SQLite file")
    parser.add_argument("-v", "--verbosity", action="count", default=0, help="increase output verbosity")

    args = parser.parse_args()

    # Our logger is the root logger now
    global logger
    logger = logging.getLogger()

    # Don't filter on level in the root logger
    logger.setLevel(logging.NOTSET)

    # Output to sys.stdout
    stdout_handler = logging.StreamHandler(stream=sys.stdout)

    # Set level according to verbosity
    if args.verbosity >= 3:
        stdout_handler.setLevel(logging.DEBUG)
    elif args.verbosity == 2:
        stdout_handler.setLevel(logging.INFO)
    elif args.verbosity >= 1:
        stdout_handler.setLevel(logging.WARNING)
    else:
        stdout_handler.setLevel(logging.CRITICAL)

    logger.addHandler(stdout_handler)

    logger.info("Reading assignments from CSV file {}".format(args.source))
    csv_mtime = os.stat(args.source).st_mtime_ns
    logger.debug("CSV file modification time: {} ns".format(csv_mtime))
    assignments = CSVBasedFixedAssignmentOptionHandler.parse_csv_file(args.source)

    logger.info("Writing assignments to SQLite file {}".format(args.destination))
    db = sqlite3.connect(args.destination, isolation_level="IMMEDIATE")
    cur = db.cursor()
    cur.execute(
        "CREATE TABLE IF NOT EXISTS assignments ("
        "id TEXT NOT NULL PRIMARY KEY, "
        "address TEXT, "
        "prefix TEXT, "
        "csv_mtime INT NOT NULL"
        ") WITHOUT ROWID"
    )

    executed_in_transaction = 0
    for key, value in assignments:
        if executed_in_transaction == 0:
            # New transaction, check if we have a newer competing update process
            cur.execute("BEGIN IMMEDIATE")
            row = cur.execute("SELECT MAX(csv_mtime) FROM assignments").fetchone()
            if row[0] and row[0] > csv_mtime:
                logger.critical("Update with newer CSV file detected, aborting")
                return 1

        cur.execute(
            "INSERT OR REPLACE INTO assignments (id, address, prefix, csv_mtime) VALUES (?, ?, ?, ?)",
            (key, str(value.address), str(value.prefix), csv_mtime),
        )

        executed_in_transaction += 1
        if executed_in_transaction >= 50:
            logger.debug("Interim commit to allow readers to access data")
            db.commit()
            time.sleep(0.05)
            executed_in_transaction = 0

    db.commit()

    cur.execute("SELECT COUNT(1) FROM assignments WHERE csv_mtime=?", [csv_mtime])
    logger.info("Wrote {} assignments".format(cur.fetchone()[0]))

    cur.execute("DELETE FROM assignments WHERE csv_mtime<?", [csv_mtime])
    logger.info("Deleted {} old assignments".format(cur.rowcount))

    db.commit()