def include_package(pkg_name, pkg_dir, include_files, include_list,
source_list):
import amalgamation
original_path = sys.path
# append the directory
sys.path.append(pkg_dir)
ext_pkg = __import__(pkg_name + '_config')
ext_include_dirs = ext_pkg.include_directories
ext_source_files = ext_pkg.source_files
include_files += amalgamation.list_includes_files(ext_include_dirs)
include_list += ext_include_dirs
source_list += ext_source_files
sys.path = original_path
sys.path.append(os.path.join(script_path, '..', '..', 'scripts'))
import package_build
(source_list, include_list, original_sources) = package_build.build_package(os.path.join(script_path, 'duckdb'), extensions)
duckdb_sources = [os.path.sep.join(package_build.get_relative_path(script_path, x).split('/')) for x in source_list]
duckdb_sources.sort()
original_sources = [os.path.join('duckdb', x) for x in original_sources]
duckdb_includes = [os.path.join('duckdb', x) for x in include_list]
duckdb_includes += ['duckdb']
# gather the include files
import amalgamation
header_files = amalgamation.list_includes_files(duckdb_includes)
# write the source list, include list and git hash to separate files
with open_utf8('sources.list', 'w+') as f:
for source_file in duckdb_sources:
f.write(source_file + "\n")
with open_utf8('includes.list', 'w+') as f:
for include_file in duckdb_includes:
f.write(include_file + '\n')
extra_files = ['sources.list', 'includes.list'] + original_sources
else:
# if amalgamation does not exist, we are in a package distribution
# read the include files, source list and include files from the supplied lists
with open_utf8('sources.list', 'r') as f:
def build_package(target_dir, linenumbers = False):
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
scripts_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(scripts_dir)
import amalgamation
sys.path.append(os.path.join(scripts_dir, '..', 'extension', 'parquet'))
import parquet_amalgamation
prev_wd = os.getcwd()
os.chdir(os.path.join(scripts_dir, '..'))
# obtain the list of source files from the amalgamation
source_list = amalgamation.list_sources()
include_list = amalgamation.list_include_dirs()
include_files = amalgamation.list_includes()
def copy_file(src, target_dir):
# get the path
full_path = src.split(os.path.sep)
current_path = target_dir
for i in range(len(full_path) - 1):
current_path = os.path.join(current_path, full_path[i])
if not os.path.isdir(current_path):
os.mkdir(current_path)
target_name = full_path[-1]
target_file = os.path.join(current_path, target_name)
amalgamation.copy_if_different(src, target_file)
# now do the same for the parquet extension
parquet_include_directories = parquet_amalgamation.include_directories
include_files += amalgamation.list_includes_files(parquet_include_directories)
include_list += parquet_include_directories
source_list += parquet_amalgamation.source_files
for src in source_list:
copy_file(src, target_dir)
for inc in include_files:
copy_file(inc, target_dir)
# handle pragma_version.cpp: paste #define DUCKDB_SOURCE_ID there
# read the source id
proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.join(scripts_dir, '..'))
githash = proc.stdout.read().strip().decode('utf8')
# open the file and read the current contents
fpath = os.path.join(target_dir, 'src', 'function', 'table', 'version', 'pragma_version.cpp')
with open(fpath, 'r') as f:
text = f.read()
# now add the DUCKDB_SOURCE_ID define, if it is not there already
found = False
lines = text.split('\n')
for i in range(len(lines)):
if '#define DUCKDB_SOURCE_ID ' in lines[i]:
lines[i] = '#define DUCKDB_SOURCE_ID "{}"'.format(githash)
found = True
break
if not found:
text = '#ifndef DUCKDB_SOURCE_ID\n#define DUCKDB_SOURCE_ID "{}"\n#endif\n'.format(githash) + text
else:
text = '\n'.join(text)
with open(fpath, 'w+') as f:
f.write(text)
def file_is_excluded(fname):
for entry in excluded_objects:
if entry in fname:
return True
return False
def generate_unity_build(entries, idx, linenumbers):
ub_file = os.path.join(target_dir, 'amalgamation-{}.cpp'.format(str(idx)))
with open(ub_file, 'w+') as f:
for entry in entries:
if linenumbers:
f.write('#line 0 "{}"\n'.format(convert_backslashes(entry)))
f.write('#include "{}"\n\n'.format(convert_backslashes(entry)))
return ub_file
def generate_unity_builds(source_list, nsplits, linenumbers):
source_list.sort()
files_per_split = len(source_list) / nsplits
new_source_files = []
current_files = []
idx = 1
for entry in source_list:
if not entry.startswith('src'):
new_source_files.append(os.path.join('duckdb', entry))
continue
current_files.append(entry)
if len(current_files) > files_per_split:
new_source_files.append(generate_unity_build(current_files, idx, linenumbers))
current_files = []
idx += 1
if len(current_files) > 0:
new_source_files.append(generate_unity_build(current_files, idx, linenumbers))
current_files = []
idx += 1
return new_source_files
original_sources = source_list
source_list = generate_unity_builds(source_list, 8, linenumbers)
os.chdir(prev_wd)
return ([convert_backslashes(x) for x in source_list if not file_is_excluded(x)],
[convert_backslashes(x) for x in include_list],
[convert_backslashes(x) for x in original_sources])
def build_package(target_dir):
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
scripts_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(scripts_dir)
import amalgamation
sys.path.append(os.path.join(scripts_dir, '..', 'extension', 'parquet'))
import parquet_amalgamation
prev_wd = os.getcwd()
os.chdir(os.path.join(scripts_dir, '..'))
# read the source id
proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=os.path.join(scripts_dir, '..'))
githash = proc.stdout.read().strip().decode('utf8')
# obtain the list of source files from the amalgamation
source_list = amalgamation.list_sources()
include_list = amalgamation.list_include_dirs()
include_files = amalgamation.list_includes()
def copy_file(src, target_dir):
# get the path
full_path = src.split(os.path.sep)
current_path = target_dir
for i in range(len(full_path) - 1):
current_path = os.path.join(current_path, full_path[i])
if not os.path.isdir(current_path):
os.mkdir(current_path)
amalgamation.copy_if_different(
src, os.path.join(current_path, full_path[-1]))
# now do the same for the parquet extension
parquet_include_directories = parquet_amalgamation.include_directories
include_files += amalgamation.list_includes_files(
parquet_include_directories)
include_list += parquet_include_directories
source_list += parquet_amalgamation.source_files
for src in source_list:
copy_file(src, target_dir)
for inc in include_files:
copy_file(inc, target_dir)
def file_is_excluded(fname):
for entry in excluded_objects:
if entry in fname:
return True
return False
def convert_backslashes(x):
return '/'.join(x.split(os.path.sep))
def generate_unity_build(entries, idx):
ub_file = os.path.join(target_dir,
'amalgamation-{}.cpp'.format(str(idx)))
with open(ub_file, 'w+') as f:
for entry in entries:
f.write('#line 0 "{}"\n'.format(convert_backslashes(entry)))
f.write('#include "{}"\n\n'.format(convert_backslashes(entry)))
return ub_file
def generate_unity_builds(source_list, nsplits):
source_list.sort()
files_per_split = len(source_list) / nsplits
new_source_files = []
current_files = []
idx = 1
for entry in source_list:
if not entry.startswith('src'):
new_source_files.append(os.path.join('duckdb', entry))
continue
current_files.append(entry)
if len(current_files) > files_per_split:
new_source_files.append(
generate_unity_build(current_files, idx))
current_files = []
idx += 1
if len(current_files) > 0:
new_source_files.append(generate_unity_build(current_files, idx))
current_files = []
idx += 1
return new_source_files
original_sources = source_list
source_list = generate_unity_builds(source_list, 8)
os.chdir(prev_wd)
return ([
convert_backslashes(x) for x in source_list if not file_is_excluded(x)
], [convert_backslashes(x) for x in include_list],
[convert_backslashes(x) for x in original_sources], githash)
