Beispiel #1
0
def check_link_table(archive_name, table, database=DATABASE,
                     processor=None,
                     columns=(('source', True), ('target', True)),
                     **extract_params):
    """Intialize a SQL table to host link tuples from dump"""
    if table in select(SQL_LIST_TABLES).split():
        logging.info("Table '%s' exists: skipping init from archive '%s'",
                     table, archive_name)
        return False
    logging.info("Loading link table '%s' with tuples from archive '%s'",
                 table, archive_name)

    query = "CREATE TABLE " + table
    query += " ("
    query += ", ".join(column + " varchar(300)" for column, _ in columns)
    query += ");"
    execute(query, database=database)

    tuples = db.extract_link(db.fetch(archive_name), **extract_params)
    if processor is not None:
        tuples = processor(tuples)
    copy(tuples, table, database=database)

    for column, index in columns:
        logging.info("Creating index on column '%s' in table '%s'",
                     column, table)
        execute(CREATE_INDEX.format(table=table, column=column))
    return True
Beispiel #2
0
def check_text_table(archive_name, table, database=DATABASE, **extract_params):
    """Intialize a SQL table to host link tuples from dump"""
    if table in select(SQL_LIST_TABLES).split():
        logging.info("Table '%s' exists: skipping init from archive '%s'",
                     table, archive_name)
        return False
    logging.info("Loading text table '%s' with tuples from archive '%s'",
                 table, archive_name)

    query = "CREATE TABLE " + table
    query += " ("
    query += " id varchar(300),"
    query += " title varchar(300),"
    query += " text text,"
    query += " lang char(2)"
    query += ");"
    execute(query, database=database)

    tuples = db.extract_text(db.fetch(archive_name), **extract_params)
    copy(tuples, table, database=database)
    logging.info("Creating index on column '%s' in table '%s'",
                 "id", table)
    execute(CREATE_INDEX.format(table=table, column="id"), database=database)
    return True