def regen_pcbuild(modules): projlines = [] filterlines = [] corelines = [] for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) deepbase = "df." + src.id deepoutfile = f"Python\\deepfreeze\\{deepbase}.c" intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h' deepintfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.c' projlines.append(f' <None Include="..\\{pyfile}">') projlines.append(f' <ModName>{src.frozenid}</ModName>') projlines.append(f' <IntFile>$(IntDir){intfile}</IntFile>') projlines.append(f' <OutFile>$(PySourcePath){header}</OutFile>') projlines.append( f' <DeepIntFile>$(IntDir){deepintfile}</DeepIntFile>') projlines.append( f' <DeepOutFile>$(PySourcePath){deepoutfile}</DeepOutFile>') projlines.append(f' </None>') filterlines.append(f' <None Include="..\\{pyfile}">') filterlines.append(' <Filter>Python Files</Filter>') filterlines.append(' </None>') corelines.append(f' <ClCompile Include="..\\{deepoutfile}" />') print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, '<!-- BEGIN frozen modules -->', '<!-- END frozen modules -->', projlines, PCBUILD_PROJECT, ) outfile.writelines(lines) print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, '<!-- BEGIN frozen modules -->', '<!-- END frozen modules -->', filterlines, PCBUILD_FILTERS, ) outfile.writelines(lines) print(f'# Updating {os.path.relpath(PCBUILD_PYTHONCORE)}') with updating_file_with_tmpfile(PCBUILD_PYTHONCORE) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, '<!-- BEGIN deepfreeze -->', '<!-- END deepfreeze -->', corelines, PCBUILD_FILTERS, ) outfile.writelines(lines)
def regen_makefile(modules): frozenfiles = [] rules = [''] for src in _iter_sources(modules): header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) frozenfiles.append(f'\t\t{header} \\') pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) # Note that we freeze the module to the target .h file # instead of going through an intermediate file like we used to. rules.append(f'{header}: Programs/_freeze_module {pyfile}') rules.append((f'\tPrograms/_freeze_module {src.frozenid} ' f'$(srcdir)/{pyfile} $(srcdir)/{header}')) rules.append('') frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") print(f'# Updating {os.path.relpath(MAKEFILE)}') with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, "FROZEN_FILES =", "# End FROZEN_FILES", frozenfiles, MAKEFILE, ) lines = replace_block( lines, "# BEGIN: freezing modules", "# END: freezing modules", rules, MAKEFILE, ) outfile.writelines(lines)
def regen_frozen(modules): headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) for src in _iter_sources(modules): # Adding a comment to separate sections here doesn't add much, # so we don't. header = relpath_for_posix_display(src.frozenfile, parentdir) headerlines.append(f'#include "{header}"') deflines = [] indent = ' ' lastsection = None for mod in modules: if mod.section != lastsection: if lastsection is not None: deflines.append('') deflines.append(f'/* {mod.section} */') lastsection = mod.section symbol = mod.symbol pkg = '-' if mod.ispkg else '' line = ('{"%s", %s, %s(int)sizeof(%s)},' ) % (mod.name, symbol, pkg, symbol) # TODO: Consider not folding lines if len(line) < 80: deflines.append(line) else: line1, _, line2 = line.rpartition(' ') deflines.append(line1) deflines.append(indent + line2) if not deflines[0]: del deflines[0] for i, line in enumerate(deflines): if line: deflines[i] = indent + line print(f'# Updating {os.path.relpath(FROZEN_FILE)}') with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile): lines = infile.readlines() # TODO: Use more obvious markers, e.g. # $START GENERATED FOOBAR$ / $END GENERATED FOOBAR$ lines = replace_block( lines, "/* Includes for frozen modules: */", "/* End includes */", headerlines, FROZEN_FILE, ) lines = replace_block( lines, "static const struct _frozen _PyImport_FrozenModules[] =", "/* sentinel */", deflines, FROZEN_FILE, ) outfile.writelines(lines)
def regen_pcbuild(frozenids, frozen): projlines = [] filterlines = [] for frozenid in frozenids: pyfile, frozenfile = frozen[frozenid] _pyfile = os.path.relpath(pyfile, ROOT_DIR).replace('/', '\\') header = os.path.relpath(frozenfile, ROOT_DIR).replace('/', '\\') intfile = header.split('\\')[-1].strip('.h') + '.g.h' projlines.append(f' <None Include="..\\{_pyfile}">') projlines.append(f' <ModName>{frozenid}</ModName>') projlines.append(f' <IntFile>$(IntDir){intfile}</IntFile>') projlines.append(f' <OutFile>$(PySourcePath){header}</OutFile>') projlines.append(f' </None>') filterlines.append(f' <None Include="..\\{_pyfile}">') filterlines.append(' <Filter>Python Files</Filter>') filterlines.append(' </None>') print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, '<!-- BEGIN frozen modules -->', '<!-- END frozen modules -->', projlines, PCBUILD_PROJECT, ) outfile.writelines(lines) print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, '<!-- BEGIN frozen modules -->', '<!-- END frozen modules -->', filterlines, PCBUILD_FILTERS, ) outfile.writelines(lines)
def regen_makefile(modules): pyfiles = [] frozenfiles = [] rules = [''] for src in _iter_sources(modules): header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) frozenfiles.append(f'\t\t{header} \\') pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) pyfiles.append(f'\t\t{pyfile} \\') freeze = (f'Programs/_freeze_module {src.frozenid} ' f'$(srcdir)/{pyfile} $(srcdir)/{header}') rules.extend([ f'{header}: Programs/_freeze_module {pyfile}', f'\t{freeze}', '', ]) pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") print(f'# Updating {os.path.relpath(MAKEFILE)}') with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, "FROZEN_FILES_IN =", "# End FROZEN_FILES_IN", pyfiles, MAKEFILE, ) lines = replace_block( lines, "FROZEN_FILES_OUT =", "# End FROZEN_FILES_OUT", frozenfiles, MAKEFILE, ) lines = replace_block( lines, "# BEGIN: freezing modules", "# END: freezing modules", rules, MAKEFILE, ) outfile.writelines(lines)
def regen_makefile(frozenids, frozen): frozenfiles = [] rules = [''] for frozenid in frozenids: pyfile, frozenfile = frozen[frozenid] header = os.path.relpath(frozenfile, ROOT_DIR) relfile = header.replace('\\', '/') frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\') _pyfile = os.path.relpath(pyfile, ROOT_DIR) tmpfile = f'{header}.new' # Note that we freeze the module to the target .h file # instead of going through an intermediate file like we used to. rules.append( f'{header}: $(srcdir)/Programs/_freeze_module $(srcdir)/{_pyfile}') rules.append(f'\t$(srcdir)/Programs/_freeze_module {frozenid} \\') rules.append(f'\t\t$(srcdir)/{_pyfile} \\') rules.append(f'\t\t$(srcdir)/{header}') rules.append('') frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") print(f'# Updating {os.path.relpath(MAKEFILE)}') with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, "FROZEN_FILES =", "# End FROZEN_FILES", frozenfiles, MAKEFILE, ) lines = replace_block( lines, "# BEGIN: freezing modules", "# END: freezing modules", rules, MAKEFILE, ) outfile.writelines(lines)
def regen_makefile(modules): pyfiles = [] frozenfiles = [] deepfreezefiles = [] rules = [''] deepfreezerules = [''] for src in _iter_sources(modules): frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) deepfreeze_header = relpath_for_posix_display(src.deepfreezefile, ROOT_DIR) frozenfiles.append(f'\t\t{frozen_header} \\') cfile = deepfreeze_header[:-2] + ".c" ofile = deepfreeze_header[:-2] + ".o" deepfreezefiles.append(f"\t\t{ofile} \\") pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) pyfiles.append(f'\t\t{pyfile} \\') if src.isbootstrap: freezecmd = '$(FREEZE_MODULE_BOOTSTRAP)' freezedep = '$(FREEZE_MODULE_BOOTSTRAP_DEPS)' else: freezecmd = '$(FREEZE_MODULE)' freezedep = '$(FREEZE_MODULE_DEPS)' freeze = (f'{freezecmd} {src.frozenid} ' f'$(srcdir)/{pyfile} {frozen_header}') rules.extend([ f'{frozen_header}: {pyfile} {freezedep}', f'\t{freeze}', '', ]) deepfreezerules.append(f'{cfile}: {frozen_header} $(DEEPFREEZE_DEPS)') deepfreezerules.append(f"\t$(PYTHON_FOR_FREEZE) " f"$(srcdir)/Tools/scripts/deepfreeze.py " f"{frozen_header} -m {src.frozenid} -o {cfile}") deepfreezerules.append('') pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") deepfreezefiles[-1] = deepfreezefiles[-1].rstrip(" \\") print(f'# Updating {os.path.relpath(MAKEFILE)}') with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, "FROZEN_FILES_IN =", "# End FROZEN_FILES_IN", pyfiles, MAKEFILE, ) lines = replace_block( lines, "FROZEN_FILES_OUT =", "# End FROZEN_FILES_OUT", frozenfiles, MAKEFILE, ) lines = replace_block( lines, "DEEPFREEZE_OBJS =", "# End DEEPFREEZE_OBJS", deepfreezefiles, MAKEFILE, ) lines = replace_block( lines, "# BEGIN: freezing modules", "# END: freezing modules", rules, MAKEFILE, ) lines = replace_block( lines, "# BEGIN: deepfreeze modules", "# END: deepfreeze modules", deepfreezerules, MAKEFILE, ) outfile.writelines(lines)
def regen_frozen(modules): headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) for src in _iter_sources(modules): # Adding a comment to separate sections here doesn't add much, # so we don't. header = relpath_for_posix_display(src.frozenfile, parentdir) headerlines.append(f'#include "{header}"') externlines = [] bootstraplines = [] stdliblines = [] testlines = [] aliaslines = [] indent = ' ' lastsection = None for mod in modules: if mod.isbootstrap: lines = bootstraplines elif mod.section == TESTS_SECTION: lines = testlines else: lines = stdliblines if mod.section != lastsection: if lastsection is not None: lines.append('') lines.append(f'/* {mod.section} */') lastsection = mod.section # Also add a extern declaration for the corresponding # deepfreeze-generated function. orig_name = mod.source.id code_name = orig_name.replace(".", "_") get_code_name = "_Py_get_%s_toplevel" % code_name externlines.append("extern PyObject *%s(void);" % get_code_name) symbol = mod.symbol pkg = '-' if mod.ispkg else '' line = ('{"%s", %s, %s(int)sizeof(%s), GET_CODE(%s)},') % ( mod.name, symbol, pkg, symbol, code_name) lines.append(line) if mod.isalias: if not mod.orig: entry = '{"%s", NULL},' % (mod.name, ) elif mod.source.ispkg: entry = '{"%s", "<%s"},' % (mod.name, mod.orig) else: entry = '{"%s", "%s"},' % (mod.name, mod.orig) aliaslines.append(indent + entry) for lines in (bootstraplines, stdliblines, testlines): # TODO: Is this necessary any more? if not lines[0]: del lines[0] for i, line in enumerate(lines): if line: lines[i] = indent + line print(f'# Updating {os.path.relpath(FROZEN_FILE)}') with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile): lines = infile.readlines() # TODO: Use more obvious markers, e.g. # $START GENERATED FOOBAR$ / $END GENERATED FOOBAR$ lines = replace_block( lines, "/* Includes for frozen modules: */", "/* End includes */", headerlines, FROZEN_FILE, ) lines = replace_block( lines, "/* Start extern declarations */", "/* End extern declarations */", externlines, FROZEN_FILE, ) lines = replace_block( lines, "static const struct _frozen bootstrap_modules[] =", "/* bootstrap sentinel */", bootstraplines, FROZEN_FILE, ) lines = replace_block( lines, "static const struct _frozen stdlib_modules[] =", "/* stdlib sentinel */", stdliblines, FROZEN_FILE, ) lines = replace_block( lines, "static const struct _frozen test_modules[] =", "/* test sentinel */", testlines, FROZEN_FILE, ) lines = replace_block( lines, "const struct _module_alias aliases[] =", "/* aliases sentinel */", aliaslines, FROZEN_FILE, ) outfile.writelines(lines)
def regen_frozen(specs, dest=MODULES_DIR): if isinstance(dest, str): frozen, frozenids = resolve_frozen_files(specs, destdir) else: frozenids, frozen = dest headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) for frozenid in frozenids: # Adding a comment to separate sections here doesn't add much, # so we don't. _, frozenfile = frozen[frozenid] header = os.path.relpath(frozenfile, parentdir) headerlines.append(f'#include "{header}"') deflines = [] indent = ' ' lastsection = None for spec in specs: frozenid, _, modname, ispkg, section = spec if section != lastsection: if lastsection is not None: deflines.append('') deflines.append(f'/* {section} */') lastsection = section # This matches what we do in Programs/_freeze_module.c: name = frozenid.replace('.', '_') symbol = '_Py_M__' + name pkg = '-' if ispkg else '' line = ('{"%s", %s, %s(int)sizeof(%s)},' % (modname, symbol, pkg, symbol)) # TODO: Consider not folding lines if len(line) < 80: deflines.append(line) else: line1, _, line2 = line.rpartition(' ') deflines.append(line1) deflines.append(indent + line2) if not deflines[0]: del deflines[0] for i, line in enumerate(deflines): if line: deflines[i] = indent + line print(f'# Updating {os.path.relpath(FROZEN_FILE)}') with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile): lines = infile.readlines() # TODO: Use more obvious markers, e.g. # $START GENERATED FOOBAR$ / $END GENERATED FOOBAR$ lines = replace_block( lines, "/* Includes for frozen modules: */", "/* End includes */", headerlines, FROZEN_FILE, ) lines = replace_block( lines, "static const struct _frozen _PyImport_FrozenModules[] =", "/* sentinel */", deflines, FROZEN_FILE, ) outfile.writelines(lines)
def regen_pcbuild(modules): projlines = [] filterlines = [] corelines = [] deepfreezerules = ['\t<Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\scripts\\deepfreeze.py" ^'] for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h' projlines.append(f' <None Include="..\\{pyfile}">') projlines.append(f' <ModName>{src.frozenid}</ModName>') projlines.append(f' <IntFile>$(IntDir){intfile}</IntFile>') projlines.append(f' <OutFile>$(PySourcePath){header}</OutFile>') projlines.append(f' </None>') filterlines.append(f' <None Include="..\\{pyfile}">') filterlines.append(' <Filter>Python Files</Filter>') filterlines.append(' </None>') deepfreezerules.append(f'\t\t "$(PySourcePath){header}:{src.frozenid}" ^') deepfreezerules.append('\t\t "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' ) corelines.append(f' <ClCompile Include="..\\Python\\deepfreeze\\deepfreeze.c" />') print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, '<!-- BEGIN frozen modules -->', '<!-- END frozen modules -->', projlines, PCBUILD_PROJECT, ) outfile.writelines(lines) with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, '<!-- BEGIN deepfreeze rule -->', '<!-- END deepfreeze rule -->', deepfreezerules, PCBUILD_PROJECT, ) outfile.writelines(lines) print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, '<!-- BEGIN frozen modules -->', '<!-- END frozen modules -->', filterlines, PCBUILD_FILTERS, ) outfile.writelines(lines) print(f'# Updating {os.path.relpath(PCBUILD_PYTHONCORE)}') with updating_file_with_tmpfile(PCBUILD_PYTHONCORE) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, '<!-- BEGIN deepfreeze -->', '<!-- END deepfreeze -->', corelines, PCBUILD_FILTERS, ) outfile.writelines(lines)
def regen_makefile(modules): pyfiles = [] frozenfiles = [] deepfreezefiles = [] rules = [''] deepfreezerules = [''] # TODO: Merge the two loops for src in _iter_sources(modules): header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) relfile = header.replace('\\', '/') _pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) # TODO: This is a bit hackish xfile = relfile.replace("/frozen_modules/", "/deepfreeze/") cfile = xfile[:-2] + ".c" ofile = xfile[:-2] + ".o" deepfreezefiles.append(f"\t\t{ofile} \\") # Also add a deepfreeze rule. deepfreezerules.append(f'{cfile}: {header} $(DEEPFREEZE_DEPS)') deepfreezerules.append( f"\t$(PYTHON_FOR_REGEN) " f"$(srcdir)/Tools/scripts/deepfreeze.py " f"{header} -m {src.frozenid} -o {cfile}") deepfreezerules.append('') for src in _iter_sources(modules): header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) frozenfiles.append(f'\t\t{header} \\') pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) pyfiles.append(f'\t\t{pyfile} \\') freeze = (f'$(FREEZE_MODULE) {src.frozenid} ' f'$(srcdir)/{pyfile} {header}') rules.extend([ f'{header}: $(FREEZE_MODULE) {pyfile}', f'\t{freeze}', '', ]) pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") deepfreezefiles[-1] = deepfreezefiles[-1].rstrip(" \\") print(f'# Updating {os.path.relpath(MAKEFILE)}') with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): lines = infile.readlines() lines = replace_block( lines, "FROZEN_FILES_IN =", "# End FROZEN_FILES_IN", pyfiles, MAKEFILE, ) lines = replace_block( lines, "FROZEN_FILES_OUT =", "# End FROZEN_FILES_OUT", frozenfiles, MAKEFILE, ) lines = replace_block( lines, "DEEPFREEZE_OBJS =", "# End DEEPFREEZE_OBJS", deepfreezefiles, MAKEFILE, ) lines = replace_block( lines, "# BEGIN: freezing modules", "# END: freezing modules", rules, MAKEFILE, ) lines = replace_block( lines, "# BEGIN: deepfreeze modules", "# END: deepfreeze modules", deepfreezerules, MAKEFILE, ) outfile.writelines(lines)
def regen_frozen(modules): headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) for src in _iter_sources(modules): # Adding a comment to separate sections here doesn't add much, # so we don't. header = relpath_for_posix_display(src.frozenfile, parentdir) headerlines.append(f'#include "{header}"') bootstraplines = [] stdliblines = [] testlines = [] aliaslines = [] indent = ' ' lastsection = None for mod in modules: if mod.frozenid in BOOTSTRAP: lines = bootstraplines elif mod.section == TESTS_SECTION: lines = testlines else: lines = stdliblines if mod.section != lastsection: if lastsection is not None: lines.append('') lines.append(f'/* {mod.section} */') lastsection = mod.section symbol = mod.symbol pkg = '-' if mod.ispkg else '' line = ('{"%s", %s, %s(int)sizeof(%s)},' ) % (mod.name, symbol, pkg, symbol) # TODO: Consider not folding lines if len(line) < 80: lines.append(line) else: line1, _, line2 = line.rpartition(' ') lines.append(line1) lines.append(indent + line2) if mod.isalias: if not mod.orig: entry = '{"%s", NULL},' % (mod.name,) elif mod.source.ispkg: entry = '{"%s", "<%s"},' % (mod.name, mod.orig) else: entry = '{"%s", "%s"},' % (mod.name, mod.orig) aliaslines.append(indent + entry) for lines in (bootstraplines, stdliblines, testlines): # TODO: Is this necessary any more? if not lines[0]: del lines[0] for i, line in enumerate(lines): if line: lines[i] = indent + line print(f'# Updating {os.path.relpath(FROZEN_FILE)}') with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile): lines = infile.readlines() # TODO: Use more obvious markers, e.g. # $START GENERATED FOOBAR$ / $END GENERATED FOOBAR$ lines = replace_block( lines, "/* Includes for frozen modules: */", "/* End includes */", headerlines, FROZEN_FILE, ) lines = replace_block( lines, "static const struct _frozen bootstrap_modules[] =", "/* bootstrap sentinel */", bootstraplines, FROZEN_FILE, ) lines = replace_block( lines, "static const struct _frozen stdlib_modules[] =", "/* stdlib sentinel */", stdliblines, FROZEN_FILE, ) lines = replace_block( lines, "static const struct _frozen test_modules[] =", "/* test sentinel */", testlines, FROZEN_FILE, ) lines = replace_block( lines, "const struct _module_alias aliases[] =", "/* aliases sentinel */", aliaslines, FROZEN_FILE, ) outfile.writelines(lines)