def build_package(target_dir, linenumbers = False):
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
scripts_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(scripts_dir)
import amalgamation
sys.path.append(os.path.join(scripts_dir, '..', 'extension', 'parquet'))
import parquet_amalgamation
prev_wd = os.getcwd()
os.chdir(os.path.join(scripts_dir, '..'))
# obtain the list of source files from the amalgamation
source_list = amalgamation.list_sources()
include_list = amalgamation.list_include_dirs()
include_files = amalgamation.list_includes()
def copy_file(src, target_dir):
# get the path
full_path = src.split(os.path.sep)
current_path = target_dir
for i in range(len(full_path) - 1):
current_path = os.path.join(current_path, full_path[i])
if not os.path.isdir(current_path):
os.mkdir(current_path)
target_name = full_path[-1]
target_file = os.path.join(current_path, target_name)
amalgamation.copy_if_different(src, target_file)
# now do the same for the parquet extension
parquet_include_directories = parquet_amalgamation.include_directories
include_files += amalgamation.list_includes_files(parquet_include_directories)
include_list += parquet_include_directories
source_list += parquet_amalgamation.source_files
for src in source_list:
copy_file(src, target_dir)
for inc in include_files:
copy_file(inc, target_dir)
# handle pragma_version.cpp: paste #define DUCKDB_SOURCE_ID there
# read the source id
proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.join(scripts_dir, '..'))
githash = proc.stdout.read().strip().decode('utf8')
# open the file and read the current contents
fpath = os.path.join(target_dir, 'src', 'function', 'table', 'version', 'pragma_version.cpp')
with open(fpath, 'r') as f:
text = f.read()
# now add the DUCKDB_SOURCE_ID define, if it is not there already
found = False
lines = text.split('\n')
for i in range(len(lines)):
if '#define DUCKDB_SOURCE_ID ' in lines[i]:
lines[i] = '#define DUCKDB_SOURCE_ID "{}"'.format(githash)
found = True
break
if not found:
text = '#ifndef DUCKDB_SOURCE_ID\n#define DUCKDB_SOURCE_ID "{}"\n#endif\n'.format(githash) + text
else:
text = '\n'.join(text)
with open(fpath, 'w+') as f:
f.write(text)
def file_is_excluded(fname):
for entry in excluded_objects:
if entry in fname:
return True
return False
def generate_unity_build(entries, idx, linenumbers):
ub_file = os.path.join(target_dir, 'amalgamation-{}.cpp'.format(str(idx)))
with open(ub_file, 'w+') as f:
for entry in entries:
if linenumbers:
f.write('#line 0 "{}"\n'.format(convert_backslashes(entry)))
f.write('#include "{}"\n\n'.format(convert_backslashes(entry)))
return ub_file
def generate_unity_builds(source_list, nsplits, linenumbers):
source_list.sort()
files_per_split = len(source_list) / nsplits
new_source_files = []
current_files = []
idx = 1
for entry in source_list:
if not entry.startswith('src'):
new_source_files.append(os.path.join('duckdb', entry))
continue
current_files.append(entry)
if len(current_files) > files_per_split:
new_source_files.append(generate_unity_build(current_files, idx, linenumbers))
current_files = []
idx += 1
if len(current_files) > 0:
new_source_files.append(generate_unity_build(current_files, idx, linenumbers))
current_files = []
idx += 1
return new_source_files
original_sources = source_list
source_list = generate_unity_builds(source_list, 8, linenumbers)
os.chdir(prev_wd)
return ([convert_backslashes(x) for x in source_list if not file_is_excluded(x)],
[convert_backslashes(x) for x in include_list],
[convert_backslashes(x) for x in original_sources])
def build_package(target_dir, extensions, linenumbers=False):
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
scripts_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(scripts_dir)
import amalgamation
prev_wd = os.getcwd()
os.chdir(os.path.join(scripts_dir, '..'))
# obtain the list of source files from the amalgamation
source_list = amalgamation.list_sources()
include_list = amalgamation.list_include_dirs()
include_files = amalgamation.list_includes()
def copy_file(src, target_dir):
# get the path
full_path = src.split(os.path.sep)
current_path = target_dir
for i in range(len(full_path) - 1):
current_path = os.path.join(current_path, full_path[i])
if not os.path.isdir(current_path):
os.mkdir(current_path)
target_name = full_path[-1]
target_file = os.path.join(current_path, target_name)
amalgamation.copy_if_different(src, target_file)
# include the main extension helper
include_files += [os.path.join('extension', 'extension_helper.hpp')]
# include the separate extensions
for ext in extensions:
ext_path = os.path.join(scripts_dir, '..', 'extension', ext)
include_package(ext, ext_path, include_files, include_list,
source_list)
for src in source_list:
copy_file(src, target_dir)
for inc in include_files:
copy_file(inc, target_dir)
# handle pragma_version.cpp: paste #define DUCKDB_SOURCE_ID and DUCKDB_VERSION there
curdir = os.getcwd()
os.chdir(os.path.join(scripts_dir, '..'))
githash = git_commit_hash()
dev_version = git_dev_version()
os.chdir(curdir)
# open the file and read the current contents
fpath = os.path.join(target_dir, 'src', 'function', 'table', 'version',
'pragma_version.cpp')
with open_utf8(fpath, 'r') as f:
text = f.read()
# now add the DUCKDB_SOURCE_ID define, if it is not there already
found_hash = False
found_dev = False
lines = text.split('\n')
for i in range(len(lines)):
if '#define DUCKDB_SOURCE_ID ' in lines[i]:
lines[i] = '#define DUCKDB_SOURCE_ID "{}"'.format(githash)
found_hash = True
break
if '#define DUCKDB_VERSION ' in lines[i]:
lines[i] = '#define DUCKDB_VERSION "{}"'.format(dev_version)
found_dev = True
break
if not found_hash:
lines = [
'#ifndef DUCKDB_SOURCE_ID',
'#define DUCKDB_SOURCE_ID "{}"'.format(githash), '#endif'
] + lines
if not found_dev:
lines = [
'#ifndef DUCKDB_VERSION',
'#define DUCKDB_VERSION "{}"'.format(dev_version), '#endif'
] + lines
text = '\n'.join(lines)
with open_utf8(fpath, 'w+') as f:
f.write(text)
def file_is_excluded(fname):
for entry in excluded_objects:
if entry in fname:
return True
return False
def generate_unity_build(entries, idx, linenumbers):
ub_file = os.path.join(target_dir,
'amalgamation-{}.cpp'.format(str(idx)))
with open_utf8(ub_file, 'w+') as f:
for entry in entries:
if linenumbers:
f.write('#line 0 "{}"\n'.format(
convert_backslashes(entry)))
f.write('#include "{}"\n\n'.format(convert_backslashes(entry)))
return ub_file
def generate_unity_builds(source_list, nsplits, linenumbers):
source_list.sort()
files_per_split = len(source_list) / nsplits
new_source_files = []
current_files = []
idx = 1
for entry in source_list:
if not entry.startswith('src'):
new_source_files.append(os.path.join('duckdb', entry))
continue
current_files.append(entry)
if len(current_files) > files_per_split:
new_source_files.append(
generate_unity_build(current_files, idx, linenumbers))
current_files = []
idx += 1
if len(current_files) > 0:
new_source_files.append(
generate_unity_build(current_files, idx, linenumbers))
current_files = []
idx += 1
return new_source_files
original_sources = source_list
source_list = generate_unity_builds(source_list, 8, linenumbers)
os.chdir(prev_wd)
return ([
convert_backslashes(x) for x in source_list if not file_is_excluded(x)
], [convert_backslashes(x) for x in include_list],
[convert_backslashes(x) for x in original_sources])
def build_package(target_dir):
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
scripts_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(scripts_dir)
import amalgamation
sys.path.append(os.path.join(scripts_dir, '..', 'extension', 'parquet'))
import parquet_amalgamation
prev_wd = os.getcwd()
os.chdir(os.path.join(scripts_dir, '..'))
# read the source id
proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=os.path.join(scripts_dir, '..'))
githash = proc.stdout.read().strip().decode('utf8')
# obtain the list of source files from the amalgamation
source_list = amalgamation.list_sources()
include_list = amalgamation.list_include_dirs()
include_files = amalgamation.list_includes()
def copy_file(src, target_dir):
# get the path
full_path = src.split(os.path.sep)
current_path = target_dir
for i in range(len(full_path) - 1):
current_path = os.path.join(current_path, full_path[i])
if not os.path.isdir(current_path):
os.mkdir(current_path)
amalgamation.copy_if_different(
src, os.path.join(current_path, full_path[-1]))
# now do the same for the parquet extension
parquet_include_directories = parquet_amalgamation.include_directories
include_files += amalgamation.list_includes_files(
parquet_include_directories)
include_list += parquet_include_directories
source_list += parquet_amalgamation.source_files
for src in source_list:
copy_file(src, target_dir)
for inc in include_files:
copy_file(inc, target_dir)
def file_is_excluded(fname):
for entry in excluded_objects:
if entry in fname:
return True
return False
def convert_backslashes(x):
return '/'.join(x.split(os.path.sep))
def generate_unity_build(entries, idx):
ub_file = os.path.join(target_dir,
'amalgamation-{}.cpp'.format(str(idx)))
with open(ub_file, 'w+') as f:
for entry in entries:
f.write('#line 0 "{}"\n'.format(convert_backslashes(entry)))
f.write('#include "{}"\n\n'.format(convert_backslashes(entry)))
return ub_file
def generate_unity_builds(source_list, nsplits):
source_list.sort()
files_per_split = len(source_list) / nsplits
new_source_files = []
current_files = []
idx = 1
for entry in source_list:
if not entry.startswith('src'):
new_source_files.append(os.path.join('duckdb', entry))
continue
current_files.append(entry)
if len(current_files) > files_per_split:
new_source_files.append(
generate_unity_build(current_files, idx))
current_files = []
idx += 1
if len(current_files) > 0:
new_source_files.append(generate_unity_build(current_files, idx))
current_files = []
idx += 1
return new_source_files
original_sources = source_list
source_list = generate_unity_builds(source_list, 8)
os.chdir(prev_wd)
return ([
convert_backslashes(x) for x in source_list if not file_is_excluded(x)
], [convert_backslashes(x) for x in include_list],
[convert_backslashes(x) for x in original_sources], githash)
def build_package(target_dir,
extensions,
linenumbers=False,
unity_count=32,
folder_name='duckdb'):
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
scripts_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(scripts_dir)
import amalgamation
prev_wd = os.getcwd()
os.chdir(os.path.join(scripts_dir, '..'))
# obtain the list of source files from the amalgamation
source_list = amalgamation.list_sources()
include_list = amalgamation.list_include_dirs()
include_files = amalgamation.list_includes()
def copy_file(src, target_dir):
# get the path
full_path = src.split(os.path.sep)
current_path = target_dir
for i in range(len(full_path) - 1):
current_path = os.path.join(current_path, full_path[i])
if not os.path.isdir(current_path):
os.mkdir(current_path)
target_name = full_path[-1]
target_file = os.path.join(current_path, target_name)
amalgamation.copy_if_different(src, target_file)
# include the main extension helper
include_files += [
os.path.join('src', 'include', 'duckdb', 'main',
'extension_helper.hpp')
]
# include the separate extensions
for ext in extensions:
ext_path = os.path.join(scripts_dir, '..', 'extension', ext)
include_package(ext, ext_path, include_files, include_list,
source_list)
for src in source_list:
copy_file(src, target_dir)
for inc in include_files:
copy_file(inc, target_dir)
# handle pragma_version.cpp: paste #define DUCKDB_SOURCE_ID and DUCKDB_VERSION there
curdir = os.getcwd()
os.chdir(os.path.join(scripts_dir, '..'))
githash = git_commit_hash()
dev_version = git_dev_version()
os.chdir(curdir)
# open the file and read the current contents
fpath = os.path.join(target_dir, 'src', 'function', 'table', 'version',
'pragma_version.cpp')
with open_utf8(fpath, 'r') as f:
text = f.read()
# now add the DUCKDB_SOURCE_ID define, if it is not there already
found_hash = False
found_dev = False
lines = text.split('\n')
for i in range(len(lines)):
if '#define DUCKDB_SOURCE_ID ' in lines[i]:
lines[i] = '#define DUCKDB_SOURCE_ID "{}"'.format(githash)
found_hash = True
break
if '#define DUCKDB_VERSION ' in lines[i]:
lines[i] = '#define DUCKDB_VERSION "{}"'.format(dev_version)
found_dev = True
break
if not found_hash:
lines = [
'#ifndef DUCKDB_SOURCE_ID',
'#define DUCKDB_SOURCE_ID "{}"'.format(githash), '#endif'
] + lines
if not found_dev:
lines = [
'#ifndef DUCKDB_VERSION',
'#define DUCKDB_VERSION "{}"'.format(dev_version), '#endif'
] + lines
text = '\n'.join(lines)
with open_utf8(fpath, 'w+') as f:
f.write(text)
def file_is_excluded(fname):
for entry in excluded_objects:
if entry in fname:
return True
return False
def generate_unity_build(entries, unity_name, linenumbers):
ub_file = os.path.join(target_dir, f'ub_{unity_name}.cpp')
with open_utf8(ub_file, 'w+') as f:
for entry in entries:
if linenumbers:
f.write('#line 0 "{}"\n'.format(
convert_backslashes(entry)))
f.write('#include "{}"\n\n'.format(convert_backslashes(entry)))
return ub_file
def generate_unity_builds(source_list, nsplits, linenumbers):
files_per_directory = {}
for source in source_list:
dirname = os.path.dirname(source)
if dirname not in files_per_directory:
files_per_directory[dirname] = []
files_per_directory[dirname].append(source)
new_source_files = []
for dirname in files_per_directory.keys():
current_files = files_per_directory[dirname]
cmake_file = os.path.join(dirname, 'CMakeLists.txt')
unity_build = False
if os.path.isfile(cmake_file):
with open(cmake_file, 'r') as f:
text = f.read()
if 'add_library_unity' in text:
unity_build = True
# re-order the files in the unity build so that they follow the same order as the CMake
scores = {}
filenames = [
x[0] for x in re.findall(
'([a-zA-Z0-9_]+[.](cpp|cc|c|cxx))', text)
]
score = 0
for filename in filenames:
scores[filename] = score
score += 1
current_files.sort(
key=lambda x: scores[os.path.basename(x)]
if os.path.basename(x) in scores else 99999)
if not unity_build:
new_source_files += [
os.path.join(folder_name, file) for file in current_files
]
else:
new_source_files.append(
generate_unity_build(current_files,
dirname.replace(os.path.sep, '_'),
linenumbers))
return new_source_files
original_sources = source_list
source_list = generate_unity_builds(source_list, unity_count, linenumbers)
os.chdir(prev_wd)
return ([
convert_backslashes(x) for x in source_list if not file_is_excluded(x)
], [convert_backslashes(x) for x in include_list],
[convert_backslashes(x) for x in original_sources])