Python add_files Examples

Example #1

File: simple_test.py Project: PaulBreugnot/Loacore

def main(workers=0):
    import os
    import loacore.utils.db as db
    import loacore.process.file_process as file_process

    # backup = db.database_backup()
    # backup.write_backup()

    db.download_db(db_name="new")
    file_process.add_files([os.path.join(os.path.dirname(__file__), "test_file.txt")], workers=workers)

Example #2

File: test_imdb.py Project: PaulBreugnot/Loacore

def add_all():
    import loacore.process.file_process as file_process
    import os
    from loacore.conf import DATA_PATH
    file_paths = []
    for dirpath, dirnames, filenames in os.walk(
            os.path.abspath(os.path.join(DATA_PATH, "raw", "imdb"))):
        for name in filenames:
            file_paths.append(os.path.join(dirpath, name))

    file_process.add_files(file_paths, encoding="utf8", lang="en")

Example #3

File: test_uci.py Project: PaulBreugnot/Loacore

def process(workers):
    import loacore.process.file_process as file_process
    file_paths = []
    for dirpath, dirnames, filenames in os.walk(
            os.path.abspath(os.path.join(loacore.conf.DATA_PATH, "raw",
                                         "uci"))):
        for name in filenames:
            file_paths.append(os.path.join(dirpath, name))

    file_process.add_files(file_paths,
                           encoding="utf8",
                           lang="en",
                           workers=workers)

Example #4

File: test_imdb.py Project: PaulBreugnot/Loacore

def add_test_imdb_pos(workers):
    import loacore.process.file_process as file_process
    import loacore.load.file_load as file_load
    from loacore.conf import DATA_PATH
    import os

    # file_load.clean_db()
    file_process.add_files([
        os.path.join(DATA_PATH, 'raw', 'imdb', 'test',
                     'test_imdb_labelled_pos.txt')
    ],
                           encoding='utf8',
                           lang='en',
                           workers=workers)

Example #5

File: test_db.py Project: PaulBreugnot/Loacore

def add_files_to_database():
    from loacore.conf import DATA_PATH
    file_paths = []

    # for dirpath, dirnames, filenames in os.walk(os.path.join(DATA_PATH, 'raw', 'TempAlta')):
    #     for name in filenames:
    #         print(name)
    #         file_paths.append(os.path.join(dirpath, name))
    #
    # for dirpath, dirnames, filenames in os.walk(os.path.join(DATA_PATH, 'raw', 'TempBaja')):
    #     for name in filenames:
    #         print(name)
    #         file_paths.append(os.path.join(dirpath, name))
    #
    # file_process.add_files(file_paths, lang='es')
    # file_paths.clear()

    file_load.clean_db()

    for dirpath, dirnames, filenames in os.walk(
            os.path.abspath(os.path.join(DATA_PATH, 'raw', 'uci'))):
        for name in filenames:
            print(name)
            file_paths.append(os.path.join(dirpath, name))

    file_process.add_files(file_paths, encoding='utf8', lang='en')
    file_paths.clear()

    for dirpath, dirnames, filenames in os.walk(
            os.path.abspath(os.path.join(DATA_PATH, 'raw', 'corrected'))):
        for name in filenames:
            print(name)
            file_paths.append(os.path.join(dirpath, name))

    file_process.add_files(file_paths, encoding='UTF-16LE', lang='es')
    file_paths.clear()

Example #6

from loacore.process.file_process import add_files
from loacore.conf import DATA_PATH
import os

print(DATA_PATH)
print(os.path.join(DATA_PATH, "raw/imdb/test/test_imdb_labelled_neg.txt"))
add_files([os.path.join(DATA_PATH, "raw/imdb/test/test_imdb_labelled_neg.txt")], workers=2)