コード例 #1
0
ファイル: package_build.py プロジェクト: saurabhdhupar/duckdb
def build_package(target_dir, linenumbers = False):
    if not os.path.isdir(target_dir):
        os.mkdir(target_dir)

    scripts_dir = os.path.dirname(os.path.abspath(__file__))
    sys.path.append(scripts_dir)
    import amalgamation
    sys.path.append(os.path.join(scripts_dir, '..', 'extension', 'parquet'))
    import parquet_amalgamation

    prev_wd = os.getcwd()
    os.chdir(os.path.join(scripts_dir, '..'))

    # obtain the list of source files from the amalgamation
    source_list = amalgamation.list_sources()
    include_list = amalgamation.list_include_dirs()
    include_files = amalgamation.list_includes()

    def copy_file(src, target_dir):
        # get the path
        full_path = src.split(os.path.sep)
        current_path = target_dir
        for i in range(len(full_path) - 1):
            current_path = os.path.join(current_path, full_path[i])
            if not os.path.isdir(current_path):
                os.mkdir(current_path)
        target_name = full_path[-1]
        target_file = os.path.join(current_path, target_name)
        amalgamation.copy_if_different(src, target_file)


    # now do the same for the parquet extension
    parquet_include_directories = parquet_amalgamation.include_directories

    include_files += amalgamation.list_includes_files(parquet_include_directories)

    include_list += parquet_include_directories
    source_list += parquet_amalgamation.source_files

    for src in source_list:
        copy_file(src, target_dir)

    for inc in include_files:
        copy_file(inc, target_dir)

    # handle pragma_version.cpp: paste #define DUCKDB_SOURCE_ID there
    # read the source id
    proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.join(scripts_dir, '..'))
    githash = proc.stdout.read().strip().decode('utf8')
    # open the file and read the current contents
    fpath = os.path.join(target_dir, 'src', 'function', 'table', 'version', 'pragma_version.cpp')
    with open(fpath, 'r') as f:
        text = f.read()
    # now add the DUCKDB_SOURCE_ID define, if it is not there already
    found = False
    lines = text.split('\n')
    for i in range(len(lines)):
        if '#define DUCKDB_SOURCE_ID ' in lines[i]:
            lines[i] = '#define DUCKDB_SOURCE_ID "{}"'.format(githash)
            found = True
            break
    if not found:
        text = '#ifndef DUCKDB_SOURCE_ID\n#define DUCKDB_SOURCE_ID "{}"\n#endif\n'.format(githash) + text
    else:
        text = '\n'.join(text)
    with open(fpath, 'w+') as f:
        f.write(text)

    def file_is_excluded(fname):
        for entry in excluded_objects:
            if entry in fname:
                return True
        return False

    def generate_unity_build(entries, idx, linenumbers):
        ub_file = os.path.join(target_dir, 'amalgamation-{}.cpp'.format(str(idx)))
        with open(ub_file, 'w+') as f:
            for entry in entries:
                if linenumbers:
                    f.write('#line 0 "{}"\n'.format(convert_backslashes(entry)))
                f.write('#include "{}"\n\n'.format(convert_backslashes(entry)))
        return ub_file

    def generate_unity_builds(source_list, nsplits, linenumbers):
        source_list.sort()

        files_per_split = len(source_list) / nsplits
        new_source_files = []
        current_files = []
        idx = 1
        for entry in source_list:
            if not entry.startswith('src'):
                new_source_files.append(os.path.join('duckdb', entry))
                continue

            current_files.append(entry)
            if len(current_files) > files_per_split:
                new_source_files.append(generate_unity_build(current_files, idx, linenumbers))
                current_files = []
                idx += 1
        if len(current_files) > 0:
            new_source_files.append(generate_unity_build(current_files, idx, linenumbers))
            current_files = []
            idx += 1

        return new_source_files

    original_sources = source_list
    source_list = generate_unity_builds(source_list, 8, linenumbers)

    os.chdir(prev_wd)
    return ([convert_backslashes(x) for x in source_list if not file_is_excluded(x)],
            [convert_backslashes(x) for x in include_list],
            [convert_backslashes(x) for x in original_sources])
コード例 #2
0
ファイル: package_build.py プロジェクト: zzl200012/duckdb
def build_package(target_dir, extensions, linenumbers=False):
    if not os.path.isdir(target_dir):
        os.mkdir(target_dir)

    scripts_dir = os.path.dirname(os.path.abspath(__file__))
    sys.path.append(scripts_dir)
    import amalgamation

    prev_wd = os.getcwd()
    os.chdir(os.path.join(scripts_dir, '..'))

    # obtain the list of source files from the amalgamation
    source_list = amalgamation.list_sources()
    include_list = amalgamation.list_include_dirs()
    include_files = amalgamation.list_includes()

    def copy_file(src, target_dir):
        # get the path
        full_path = src.split(os.path.sep)
        current_path = target_dir
        for i in range(len(full_path) - 1):
            current_path = os.path.join(current_path, full_path[i])
            if not os.path.isdir(current_path):
                os.mkdir(current_path)
        target_name = full_path[-1]
        target_file = os.path.join(current_path, target_name)
        amalgamation.copy_if_different(src, target_file)

    # include the main extension helper
    include_files += [os.path.join('extension', 'extension_helper.hpp')]
    # include the separate extensions
    for ext in extensions:
        ext_path = os.path.join(scripts_dir, '..', 'extension', ext)
        include_package(ext, ext_path, include_files, include_list,
                        source_list)

    for src in source_list:
        copy_file(src, target_dir)

    for inc in include_files:
        copy_file(inc, target_dir)

    # handle pragma_version.cpp: paste #define DUCKDB_SOURCE_ID and DUCKDB_VERSION there
    curdir = os.getcwd()
    os.chdir(os.path.join(scripts_dir, '..'))
    githash = git_commit_hash()
    dev_version = git_dev_version()
    os.chdir(curdir)
    # open the file and read the current contents
    fpath = os.path.join(target_dir, 'src', 'function', 'table', 'version',
                         'pragma_version.cpp')
    with open_utf8(fpath, 'r') as f:
        text = f.read()
    # now add the DUCKDB_SOURCE_ID define, if it is not there already
    found_hash = False
    found_dev = False
    lines = text.split('\n')
    for i in range(len(lines)):
        if '#define DUCKDB_SOURCE_ID ' in lines[i]:
            lines[i] = '#define DUCKDB_SOURCE_ID "{}"'.format(githash)
            found_hash = True
            break
        if '#define DUCKDB_VERSION ' in lines[i]:
            lines[i] = '#define DUCKDB_VERSION "{}"'.format(dev_version)
            found_dev = True
            break
    if not found_hash:
        lines = [
            '#ifndef DUCKDB_SOURCE_ID',
            '#define DUCKDB_SOURCE_ID "{}"'.format(githash), '#endif'
        ] + lines
    if not found_dev:
        lines = [
            '#ifndef DUCKDB_VERSION',
            '#define DUCKDB_VERSION "{}"'.format(dev_version), '#endif'
        ] + lines
    text = '\n'.join(lines)
    with open_utf8(fpath, 'w+') as f:
        f.write(text)

    def file_is_excluded(fname):
        for entry in excluded_objects:
            if entry in fname:
                return True
        return False

    def generate_unity_build(entries, idx, linenumbers):
        ub_file = os.path.join(target_dir,
                               'amalgamation-{}.cpp'.format(str(idx)))
        with open_utf8(ub_file, 'w+') as f:
            for entry in entries:
                if linenumbers:
                    f.write('#line 0 "{}"\n'.format(
                        convert_backslashes(entry)))
                f.write('#include "{}"\n\n'.format(convert_backslashes(entry)))
        return ub_file

    def generate_unity_builds(source_list, nsplits, linenumbers):
        source_list.sort()

        files_per_split = len(source_list) / nsplits
        new_source_files = []
        current_files = []
        idx = 1
        for entry in source_list:
            if not entry.startswith('src'):
                new_source_files.append(os.path.join('duckdb', entry))
                continue

            current_files.append(entry)
            if len(current_files) > files_per_split:
                new_source_files.append(
                    generate_unity_build(current_files, idx, linenumbers))
                current_files = []
                idx += 1
        if len(current_files) > 0:
            new_source_files.append(
                generate_unity_build(current_files, idx, linenumbers))
            current_files = []
            idx += 1

        return new_source_files

    original_sources = source_list
    source_list = generate_unity_builds(source_list, 8, linenumbers)

    os.chdir(prev_wd)
    return ([
        convert_backslashes(x) for x in source_list if not file_is_excluded(x)
    ], [convert_backslashes(x) for x in include_list],
            [convert_backslashes(x) for x in original_sources])
コード例 #3
0
def build_package(target_dir):
    if not os.path.isdir(target_dir):
        os.mkdir(target_dir)

    scripts_dir = os.path.dirname(os.path.abspath(__file__))
    sys.path.append(scripts_dir)
    import amalgamation
    sys.path.append(os.path.join(scripts_dir, '..', 'extension', 'parquet'))
    import parquet_amalgamation

    prev_wd = os.getcwd()
    os.chdir(os.path.join(scripts_dir, '..'))

    # read the source id
    proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE,
                            cwd=os.path.join(scripts_dir, '..'))
    githash = proc.stdout.read().strip().decode('utf8')

    # obtain the list of source files from the amalgamation
    source_list = amalgamation.list_sources()
    include_list = amalgamation.list_include_dirs()
    include_files = amalgamation.list_includes()

    def copy_file(src, target_dir):
        # get the path
        full_path = src.split(os.path.sep)
        current_path = target_dir
        for i in range(len(full_path) - 1):
            current_path = os.path.join(current_path, full_path[i])
            if not os.path.isdir(current_path):
                os.mkdir(current_path)
        amalgamation.copy_if_different(
            src, os.path.join(current_path, full_path[-1]))

    # now do the same for the parquet extension
    parquet_include_directories = parquet_amalgamation.include_directories

    include_files += amalgamation.list_includes_files(
        parquet_include_directories)

    include_list += parquet_include_directories
    source_list += parquet_amalgamation.source_files

    for src in source_list:
        copy_file(src, target_dir)

    for inc in include_files:
        copy_file(inc, target_dir)

    def file_is_excluded(fname):
        for entry in excluded_objects:
            if entry in fname:
                return True
        return False

    def convert_backslashes(x):
        return '/'.join(x.split(os.path.sep))

    def generate_unity_build(entries, idx):
        ub_file = os.path.join(target_dir,
                               'amalgamation-{}.cpp'.format(str(idx)))
        with open(ub_file, 'w+') as f:
            for entry in entries:
                f.write('#line 0 "{}"\n'.format(convert_backslashes(entry)))
                f.write('#include "{}"\n\n'.format(convert_backslashes(entry)))
        return ub_file

    def generate_unity_builds(source_list, nsplits):
        source_list.sort()

        files_per_split = len(source_list) / nsplits
        new_source_files = []
        current_files = []
        idx = 1
        for entry in source_list:
            if not entry.startswith('src'):
                new_source_files.append(os.path.join('duckdb', entry))
                continue

            current_files.append(entry)
            if len(current_files) > files_per_split:
                new_source_files.append(
                    generate_unity_build(current_files, idx))
                current_files = []
                idx += 1
        if len(current_files) > 0:
            new_source_files.append(generate_unity_build(current_files, idx))
            current_files = []
            idx += 1

        return new_source_files

    original_sources = source_list
    source_list = generate_unity_builds(source_list, 8)

    os.chdir(prev_wd)
    return ([
        convert_backslashes(x) for x in source_list if not file_is_excluded(x)
    ], [convert_backslashes(x) for x in include_list],
            [convert_backslashes(x) for x in original_sources], githash)
コード例 #4
0
def build_package(target_dir,
                  extensions,
                  linenumbers=False,
                  unity_count=32,
                  folder_name='duckdb'):
    if not os.path.isdir(target_dir):
        os.mkdir(target_dir)

    scripts_dir = os.path.dirname(os.path.abspath(__file__))
    sys.path.append(scripts_dir)
    import amalgamation

    prev_wd = os.getcwd()
    os.chdir(os.path.join(scripts_dir, '..'))

    # obtain the list of source files from the amalgamation
    source_list = amalgamation.list_sources()
    include_list = amalgamation.list_include_dirs()
    include_files = amalgamation.list_includes()

    def copy_file(src, target_dir):
        # get the path
        full_path = src.split(os.path.sep)
        current_path = target_dir
        for i in range(len(full_path) - 1):
            current_path = os.path.join(current_path, full_path[i])
            if not os.path.isdir(current_path):
                os.mkdir(current_path)
        target_name = full_path[-1]
        target_file = os.path.join(current_path, target_name)
        amalgamation.copy_if_different(src, target_file)

    # include the main extension helper
    include_files += [
        os.path.join('src', 'include', 'duckdb', 'main',
                     'extension_helper.hpp')
    ]
    # include the separate extensions
    for ext in extensions:
        ext_path = os.path.join(scripts_dir, '..', 'extension', ext)
        include_package(ext, ext_path, include_files, include_list,
                        source_list)

    for src in source_list:
        copy_file(src, target_dir)

    for inc in include_files:
        copy_file(inc, target_dir)

    # handle pragma_version.cpp: paste #define DUCKDB_SOURCE_ID and DUCKDB_VERSION there
    curdir = os.getcwd()
    os.chdir(os.path.join(scripts_dir, '..'))
    githash = git_commit_hash()
    dev_version = git_dev_version()
    os.chdir(curdir)
    # open the file and read the current contents
    fpath = os.path.join(target_dir, 'src', 'function', 'table', 'version',
                         'pragma_version.cpp')
    with open_utf8(fpath, 'r') as f:
        text = f.read()
    # now add the DUCKDB_SOURCE_ID define, if it is not there already
    found_hash = False
    found_dev = False
    lines = text.split('\n')
    for i in range(len(lines)):
        if '#define DUCKDB_SOURCE_ID ' in lines[i]:
            lines[i] = '#define DUCKDB_SOURCE_ID "{}"'.format(githash)
            found_hash = True
            break
        if '#define DUCKDB_VERSION ' in lines[i]:
            lines[i] = '#define DUCKDB_VERSION "{}"'.format(dev_version)
            found_dev = True
            break
    if not found_hash:
        lines = [
            '#ifndef DUCKDB_SOURCE_ID',
            '#define DUCKDB_SOURCE_ID "{}"'.format(githash), '#endif'
        ] + lines
    if not found_dev:
        lines = [
            '#ifndef DUCKDB_VERSION',
            '#define DUCKDB_VERSION "{}"'.format(dev_version), '#endif'
        ] + lines
    text = '\n'.join(lines)
    with open_utf8(fpath, 'w+') as f:
        f.write(text)

    def file_is_excluded(fname):
        for entry in excluded_objects:
            if entry in fname:
                return True
        return False

    def generate_unity_build(entries, unity_name, linenumbers):
        ub_file = os.path.join(target_dir, f'ub_{unity_name}.cpp')
        with open_utf8(ub_file, 'w+') as f:
            for entry in entries:
                if linenumbers:
                    f.write('#line 0 "{}"\n'.format(
                        convert_backslashes(entry)))
                f.write('#include "{}"\n\n'.format(convert_backslashes(entry)))
        return ub_file

    def generate_unity_builds(source_list, nsplits, linenumbers):
        files_per_directory = {}
        for source in source_list:
            dirname = os.path.dirname(source)
            if dirname not in files_per_directory:
                files_per_directory[dirname] = []
            files_per_directory[dirname].append(source)

        new_source_files = []
        for dirname in files_per_directory.keys():
            current_files = files_per_directory[dirname]
            cmake_file = os.path.join(dirname, 'CMakeLists.txt')
            unity_build = False
            if os.path.isfile(cmake_file):
                with open(cmake_file, 'r') as f:
                    text = f.read()
                    if 'add_library_unity' in text:
                        unity_build = True
                        # re-order the files in the unity build so that they follow the same order as the CMake
                        scores = {}
                        filenames = [
                            x[0] for x in re.findall(
                                '([a-zA-Z0-9_]+[.](cpp|cc|c|cxx))', text)
                        ]
                        score = 0
                        for filename in filenames:
                            scores[filename] = score
                            score += 1
                        current_files.sort(
                            key=lambda x: scores[os.path.basename(x)]
                            if os.path.basename(x) in scores else 99999)
            if not unity_build:
                new_source_files += [
                    os.path.join(folder_name, file) for file in current_files
                ]
            else:
                new_source_files.append(
                    generate_unity_build(current_files,
                                         dirname.replace(os.path.sep, '_'),
                                         linenumbers))
        return new_source_files

    original_sources = source_list
    source_list = generate_unity_builds(source_list, unity_count, linenumbers)

    os.chdir(prev_wd)
    return ([
        convert_backslashes(x) for x in source_list if not file_is_excluded(x)
    ], [convert_backslashes(x) for x in include_list],
            [convert_backslashes(x) for x in original_sources])