def include_package(pkg_name, pkg_dir, include_files, include_list, source_list): import amalgamation original_path = sys.path # append the directory sys.path.append(pkg_dir) ext_pkg = __import__(pkg_name + '_config') ext_include_dirs = ext_pkg.include_directories ext_source_files = ext_pkg.source_files include_files += amalgamation.list_includes_files(ext_include_dirs) include_list += ext_include_dirs source_list += ext_source_files sys.path = original_path
sys.path.append(os.path.join(script_path, '..', '..', 'scripts')) import package_build (source_list, include_list, original_sources) = package_build.build_package(os.path.join(script_path, 'duckdb'), extensions) duckdb_sources = [os.path.sep.join(package_build.get_relative_path(script_path, x).split('/')) for x in source_list] duckdb_sources.sort() original_sources = [os.path.join('duckdb', x) for x in original_sources] duckdb_includes = [os.path.join('duckdb', x) for x in include_list] duckdb_includes += ['duckdb'] # gather the include files import amalgamation header_files = amalgamation.list_includes_files(duckdb_includes) # write the source list, include list and git hash to separate files with open_utf8('sources.list', 'w+') as f: for source_file in duckdb_sources: f.write(source_file + "\n") with open_utf8('includes.list', 'w+') as f: for include_file in duckdb_includes: f.write(include_file + '\n') extra_files = ['sources.list', 'includes.list'] + original_sources else: # if amalgamation does not exist, we are in a package distribution # read the include files, source list and include files from the supplied lists with open_utf8('sources.list', 'r') as f:
def build_package(target_dir, linenumbers = False): if not os.path.isdir(target_dir): os.mkdir(target_dir) scripts_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(scripts_dir) import amalgamation sys.path.append(os.path.join(scripts_dir, '..', 'extension', 'parquet')) import parquet_amalgamation prev_wd = os.getcwd() os.chdir(os.path.join(scripts_dir, '..')) # obtain the list of source files from the amalgamation source_list = amalgamation.list_sources() include_list = amalgamation.list_include_dirs() include_files = amalgamation.list_includes() def copy_file(src, target_dir): # get the path full_path = src.split(os.path.sep) current_path = target_dir for i in range(len(full_path) - 1): current_path = os.path.join(current_path, full_path[i]) if not os.path.isdir(current_path): os.mkdir(current_path) target_name = full_path[-1] target_file = os.path.join(current_path, target_name) amalgamation.copy_if_different(src, target_file) # now do the same for the parquet extension parquet_include_directories = parquet_amalgamation.include_directories include_files += amalgamation.list_includes_files(parquet_include_directories) include_list += parquet_include_directories source_list += parquet_amalgamation.source_files for src in source_list: copy_file(src, target_dir) for inc in include_files: copy_file(inc, target_dir) # handle pragma_version.cpp: paste #define DUCKDB_SOURCE_ID there # read the source id proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.join(scripts_dir, '..')) githash = proc.stdout.read().strip().decode('utf8') # open the file and read the current contents fpath = os.path.join(target_dir, 'src', 'function', 'table', 'version', 'pragma_version.cpp') with open(fpath, 'r') as f: text = f.read() # now add the DUCKDB_SOURCE_ID define, if it is not there already found = False lines = text.split('\n') for i in range(len(lines)): if '#define DUCKDB_SOURCE_ID ' in lines[i]: lines[i] = '#define DUCKDB_SOURCE_ID "{}"'.format(githash) found = True break if not found: text = '#ifndef DUCKDB_SOURCE_ID\n#define DUCKDB_SOURCE_ID "{}"\n#endif\n'.format(githash) + text else: text = '\n'.join(text) with open(fpath, 'w+') as f: f.write(text) def file_is_excluded(fname): for entry in excluded_objects: if entry in fname: return True return False def generate_unity_build(entries, idx, linenumbers): ub_file = os.path.join(target_dir, 'amalgamation-{}.cpp'.format(str(idx))) with open(ub_file, 'w+') as f: for entry in entries: if linenumbers: f.write('#line 0 "{}"\n'.format(convert_backslashes(entry))) f.write('#include "{}"\n\n'.format(convert_backslashes(entry))) return ub_file def generate_unity_builds(source_list, nsplits, linenumbers): source_list.sort() files_per_split = len(source_list) / nsplits new_source_files = [] current_files = [] idx = 1 for entry in source_list: if not entry.startswith('src'): new_source_files.append(os.path.join('duckdb', entry)) continue current_files.append(entry) if len(current_files) > files_per_split: new_source_files.append(generate_unity_build(current_files, idx, linenumbers)) current_files = [] idx += 1 if len(current_files) > 0: new_source_files.append(generate_unity_build(current_files, idx, linenumbers)) current_files = [] idx += 1 return new_source_files original_sources = source_list source_list = generate_unity_builds(source_list, 8, linenumbers) os.chdir(prev_wd) return ([convert_backslashes(x) for x in source_list if not file_is_excluded(x)], [convert_backslashes(x) for x in include_list], [convert_backslashes(x) for x in original_sources])
def build_package(target_dir): if not os.path.isdir(target_dir): os.mkdir(target_dir) scripts_dir = os.path.dirname(os.path.abspath(__file__)) sys.path.append(scripts_dir) import amalgamation sys.path.append(os.path.join(scripts_dir, '..', 'extension', 'parquet')) import parquet_amalgamation prev_wd = os.getcwd() os.chdir(os.path.join(scripts_dir, '..')) # read the source id proc = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.join(scripts_dir, '..')) githash = proc.stdout.read().strip().decode('utf8') # obtain the list of source files from the amalgamation source_list = amalgamation.list_sources() include_list = amalgamation.list_include_dirs() include_files = amalgamation.list_includes() def copy_file(src, target_dir): # get the path full_path = src.split(os.path.sep) current_path = target_dir for i in range(len(full_path) - 1): current_path = os.path.join(current_path, full_path[i]) if not os.path.isdir(current_path): os.mkdir(current_path) amalgamation.copy_if_different( src, os.path.join(current_path, full_path[-1])) # now do the same for the parquet extension parquet_include_directories = parquet_amalgamation.include_directories include_files += amalgamation.list_includes_files( parquet_include_directories) include_list += parquet_include_directories source_list += parquet_amalgamation.source_files for src in source_list: copy_file(src, target_dir) for inc in include_files: copy_file(inc, target_dir) def file_is_excluded(fname): for entry in excluded_objects: if entry in fname: return True return False def convert_backslashes(x): return '/'.join(x.split(os.path.sep)) def generate_unity_build(entries, idx): ub_file = os.path.join(target_dir, 'amalgamation-{}.cpp'.format(str(idx))) with open(ub_file, 'w+') as f: for entry in entries: f.write('#line 0 "{}"\n'.format(convert_backslashes(entry))) f.write('#include "{}"\n\n'.format(convert_backslashes(entry))) return ub_file def generate_unity_builds(source_list, nsplits): source_list.sort() files_per_split = len(source_list) / nsplits new_source_files = [] current_files = [] idx = 1 for entry in source_list: if not entry.startswith('src'): new_source_files.append(os.path.join('duckdb', entry)) continue current_files.append(entry) if len(current_files) > files_per_split: new_source_files.append( generate_unity_build(current_files, idx)) current_files = [] idx += 1 if len(current_files) > 0: new_source_files.append(generate_unity_build(current_files, idx)) current_files = [] idx += 1 return new_source_files original_sources = source_list source_list = generate_unity_builds(source_list, 8) os.chdir(prev_wd) return ([ convert_backslashes(x) for x in source_list if not file_is_excluded(x) ], [convert_backslashes(x) for x in include_list], [convert_backslashes(x) for x in original_sources], githash)