def get_targets(base_dir): sys.path.append("%s/tools/gyp/pylib" % base_dir) import gyp path = "deps/spidershim/tests.gyp" params = { b'parallel': False, b'root_targets': None, } includes = [ "config.gypi", "common.gypi", ] default_variables = { b'OS': 'linux', b'STATIC_LIB_PREFIX': '', b'STATIC_LIB_SUFFIX': '', } # Use the gypd generator as very simple generator, since we're only # interested in reading the target names. generator, flat_list, targets, data = \ gyp.Load([path], format='gypd', \ params=params, includes=includes, \ default_variables=default_variables) external_sm = { 'Release': data['deps/spidershim/tests.gyp']['variables']['external_spidermonkey_release'], 'Debug': data['deps/spidershim/tests.gyp']['variables']['external_spidermonkey_debug'], } targets = [target['target_name'] for target in data['deps/spidershim/tests.gyp']['targets']] return (targets, external_sm)
def load_gyp(*args): _, flat_list, targets, data = gyp.Load(*args) return flat_list, targets, data
def read_from_gyp(config, path, output, vars, non_unified_sources=set()): """Read a gyp configuration and emits GypContexts for the backend to process. config is a ConfigEnvironment, path is the path to a root gyp configuration file, output is the base path under which the objdir for the various gyp dependencies will be, and vars a dict of variables to pass to the gyp processor. """ time_start = time.time() # gyp expects plain str instead of unicode. The frontend code gives us # unicode strings, so convert them. path = encode(path) str_vars = dict((name, encode(value)) for name, value in vars.items()) params = { b'parallel': False, b'generator_flags': {}, b'build_files': [path], } # Files that gyp_chromium always includes includes = [encode(mozpath.join(script_dir, 'common.gypi'))] finder = FileFinder(chrome_src, find_executables=False) includes.extend( encode(mozpath.join(chrome_src, name)) for name, _ in finder.find('*/supplement.gypi')) # Read the given gyp file and its dependencies. generator, flat_list, targets, data = \ gyp.Load([path], format=b'mozbuild', default_variables=str_vars, includes=includes, depth=encode(mozpath.dirname(path)), params=params) # Process all targets from the given gyp files and its dependencies. # The path given to AllTargets needs to use os.sep, while the frontend code # gives us paths normalized with forward slash separator. for target in gyp.common.AllTargets(flat_list, targets, path.replace(b'/', os.sep)): build_file, target_name, toolset = gyp.common.ParseQualifiedTarget( target) # Each target is given its own objdir. The base of that objdir # is derived from the relative path from the root gyp file path # to the current build_file, placed under the given output # directory. Since several targets can be in a given build_file, # separate them in subdirectories using the build_file basename # and the target_name. reldir = mozpath.relpath(mozpath.dirname(build_file), mozpath.dirname(path)) subdir = '%s_%s' % ( mozpath.splitext(mozpath.basename(build_file))[0], target_name, ) # Emit a context for each target. context = GypContext( config, mozpath.relpath(mozpath.join(output, reldir, subdir), config.topobjdir)) context.add_source(mozpath.abspath(build_file)) # The list of included files returned by gyp are relative to build_file for f in data[build_file]['included_files']: context.add_source( mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f))) spec = targets[target] # Derive which gyp configuration to use based on MOZ_DEBUG. c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release' if c not in spec['configurations']: raise RuntimeError('Missing %s gyp configuration for target %s ' 'in %s' % (c, target_name, build_file)) target_conf = spec['configurations'][c] if spec['type'] == 'none': continue elif spec['type'] == 'static_library': # Remove leading 'lib' from the target_name if any, and use as # library name. name = spec['target_name'] if name.startswith('lib'): name = name[3:] # The context expects an unicode string. context['LIBRARY_NAME'] = name.decode('utf-8') # gyp files contain headers and asm sources in sources lists. sources = [] unified_sources = [] extensions = set() for f in spec.get('sources', []): ext = mozpath.splitext(f)[-1] extensions.add(ext) s = SourcePath(context, f) if ext == '.h': continue if ext != '.S' and s not in non_unified_sources: unified_sources.append(s) else: sources.append(s) # The context expects alphabetical order when adding sources context['SOURCES'] = alphabetical_sorted(sources) context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources) for define in target_conf.get('defines', []): if '=' in define: name, value = define.split('=', 1) context['DEFINES'][name] = value else: context['DEFINES'][define] = True for include in target_conf.get('include_dirs', []): # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do. # # NB: gyp files sometimes have actual absolute paths (e.g. # /usr/include32) and sometimes paths that moz.build considers # absolute, i.e. starting from topsrcdir. There's no good way # to tell them apart here, and the actual absolute paths are # likely bogus. In any event, actual absolute paths will be # filtered out by trying to find them in topsrcdir. if include.startswith('/'): resolved = mozpath.abspath( mozpath.join(config.topsrcdir, include[1:])) else: resolved = mozpath.abspath( mozpath.join(mozpath.dirname(build_file), include)) if not os.path.exists(resolved): continue context['LOCAL_INCLUDES'] += [include] context['ASFLAGS'] = target_conf.get('asflags_mozilla', []) flags = target_conf.get('cflags_mozilla', []) if flags: suffix_map = { '.c': 'CFLAGS', '.cpp': 'CXXFLAGS', '.cc': 'CXXFLAGS', '.m': 'CMFLAGS', '.mm': 'CMMFLAGS', } variables = (suffix_map[e] for e in extensions if e in suffix_map) for var in variables: context[var].extend(flags) else: # Ignore other types than static_library because we don't have # anything using them, and we're not testing them. They can be # added when that becomes necessary. raise NotImplementedError('Unsupported gyp target type: %s' % spec['type']) # Add some features to all contexts. Put here in case LOCAL_INCLUDES # order matters. context['LOCAL_INCLUDES'] += [ '/ipc/chromium/src', '/ipc/glue', ] context['GENERATED_INCLUDES'] += ['/ipc/ipdl/_ipdlheaders'] # These get set via VC project file settings for normal GYP builds. if config.substs['OS_TARGET'] == 'WINNT': context['DEFINES']['UNICODE'] = True context['DEFINES']['_UNICODE'] = True context['DISABLE_STL_WRAPPING'] = True context.execution_time = time.time() - time_start yield context time_start = time.time()
def read_from_gyp(config, path, output, vars, non_unified_sources=set()): """Read a gyp configuration and emits GypSandboxes for the backend to process. config is a ConfigEnvironment, path is the path to a root gyp configuration file, output is the base path under which the objdir for the various gyp dependencies will be, and vars a dict of variables to pass to the gyp processor. """ time_start = time.time() all_sources = set() # gyp expects plain str instead of unicode. The frontend code gives us # unicode strings, so convert them. path = encode(path) str_vars = dict((name, encode(value)) for name, value in vars.items()) params = { b'parallel': False, b'generator_flags': {}, b'build_files': [path], } # Files that gyp_chromium always includes includes = [encode(mozpath.join(script_dir, 'common.gypi'))] finder = FileFinder(chrome_src, find_executables=False) includes.extend( encode(mozpath.join(chrome_src, name)) for name, _ in finder.find('*/supplement.gypi')) # Read the given gyp file and its dependencies. generator, flat_list, targets, data = \ gyp.Load([path], format=b'mozbuild', default_variables=str_vars, includes=includes, depth=encode(mozpath.dirname(path)), params=params) # Process all targets from the given gyp files and its dependencies. # The path given to AllTargets needs to use os.sep, while the frontend code # gives us paths normalized with forward slash separator. for target in gyp.common.AllTargets(flat_list, targets, path.replace(b'/', os.sep)): build_file, target_name, toolset = gyp.common.ParseQualifiedTarget( target) # The list of included files returned by gyp are relative to build_file included_files = [ mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f)) for f in data[build_file]['included_files'] ] # Emit a sandbox for each target. sandbox = GypSandbox(mozpath.abspath(build_file), included_files) with sandbox.allow_all_writes() as d: topsrcdir = d['TOPSRCDIR'] = config.topsrcdir d['TOPOBJDIR'] = config.topobjdir relsrcdir = d['RELATIVEDIR'] = mozpath.relpath( mozpath.dirname(build_file), config.topsrcdir) d['SRCDIR'] = mozpath.join(topsrcdir, relsrcdir) # Each target is given its own objdir. The base of that objdir # is derived from the relative path from the root gyp file path # to the current build_file, placed under the given output # directory. Since several targets can be in a given build_file, # separate them in subdirectories using the build_file basename # and the target_name. reldir = mozpath.relpath(mozpath.dirname(build_file), mozpath.dirname(path)) subdir = '%s_%s' % ( mozpath.splitext(mozpath.basename(build_file))[0], target_name, ) d['OBJDIR'] = mozpath.join(output, reldir, subdir) d['IS_GYP_DIR'] = True spec = targets[target] # Derive which gyp configuration to use based on MOZ_DEBUG. c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release' if c not in spec['configurations']: raise RuntimeError('Missing %s gyp configuration for target %s ' 'in %s' % (c, target_name, build_file)) target_conf = spec['configurations'][c] if spec['type'] == 'none': continue elif spec['type'] == 'static_library': sandbox['FORCE_STATIC_LIB'] = True # Remove leading 'lib' from the target_name if any, and use as # library name. name = spec['target_name'] if name.startswith('lib'): name = name[3:] # The sandbox expects an unicode string. sandbox['LIBRARY_NAME'] = name.decode('utf-8') # gyp files contain headers and asm sources in sources lists. sources = set( mozpath.normpath(mozpath.join(sandbox['SRCDIR'], f)) for f in spec.get('sources', []) if mozpath.splitext(f)[-1] != '.h') asm_sources = set(f for f in sources if f.endswith('.S')) unified_sources = sources - non_unified_sources - asm_sources sources -= unified_sources all_sources |= sources # The sandbox expects alphabetical order when adding sources sandbox['SOURCES'] = alphabetical_sorted(sources) sandbox['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources) for define in target_conf.get('defines', []): if '=' in define: name, value = define.split('=', 1) sandbox['DEFINES'][name] = value else: sandbox['DEFINES'][define] = True for include in target_conf.get('include_dirs', []): sandbox['LOCAL_INCLUDES'] += [include] with sandbox.allow_all_writes() as d: d['EXTRA_ASSEMBLER_FLAGS'] = target_conf.get( 'asflags_mozilla', []) d['EXTRA_COMPILE_FLAGS'] = target_conf.get( 'cflags_mozilla', []) else: # Ignore other types than static_library because we don't have # anything using them, and we're not testing them. They can be # added when that becomes necessary. raise NotImplementedError('Unsupported gyp target type: %s' % spec['type']) sandbox.execution_time = time.time() - time_start yield sandbox time_start = time.time()
def read_from_gyp(config, path, output, vars, no_chromium, no_unified, action_overrides, non_unified_sources=set()): """Read a gyp configuration and emits GypContexts for the backend to process. config is a ConfigEnvironment, path is the path to a root gyp configuration file, output is the base path under which the objdir for the various gyp dependencies will be, and vars a dict of variables to pass to the gyp processor. """ is_win = config.substs['OS_TARGET'] == 'WINNT' is_msvc = bool(config.substs['_MSC_VER']) # gyp expects plain str instead of unicode. The frontend code gives us # unicode strings, so convert them. path = encode(path) str_vars = dict((name, encode(value)) for name, value in vars.items()) if is_msvc: # This isn't actually used anywhere in this generator, but it's needed # to override the registry detection of VC++ in gyp. os.environ['GYP_MSVS_OVERRIDE_PATH'] = 'fake_path' os.environ['GYP_MSVS_VERSION'] = config.substs['MSVS_VERSION'] params = { b'parallel': False, b'generator_flags': {}, b'build_files': [path], b'root_targets': None, } if no_chromium: includes = [] depth = mozpath.dirname(path) else: depth = chrome_src # Files that gyp_chromium always includes includes = [encode(mozpath.join(script_dir, 'common.gypi'))] finder = FileFinder(chrome_src, find_executables=False) includes.extend( encode(mozpath.join(chrome_src, name)) for name, _ in finder.find('*/supplement.gypi')) # Read the given gyp file and its dependencies. generator, flat_list, targets, data = \ gyp.Load([path], format=b'mozbuild', default_variables=str_vars, includes=includes, depth=encode(depth), params=params) # Process all targets from the given gyp files and its dependencies. # The path given to AllTargets needs to use os.sep, while the frontend code # gives us paths normalized with forward slash separator. for target in gyp.common.AllTargets(flat_list, targets, path.replace(b'/', os.sep)): build_file, target_name, toolset = gyp.common.ParseQualifiedTarget( target) # Each target is given its own objdir. The base of that objdir # is derived from the relative path from the root gyp file path # to the current build_file, placed under the given output # directory. Since several targets can be in a given build_file, # separate them in subdirectories using the build_file basename # and the target_name. reldir = mozpath.relpath(mozpath.dirname(build_file), mozpath.dirname(path)) subdir = '%s_%s' % ( mozpath.splitext(mozpath.basename(build_file))[0], target_name, ) # Emit a context for each target. context = GypContext( config, mozpath.relpath(mozpath.join(output, reldir, subdir), config.topobjdir)) context.add_source(mozpath.abspath(build_file)) # The list of included files returned by gyp are relative to build_file for f in data[build_file]['included_files']: context.add_source( mozpath.abspath(mozpath.join(mozpath.dirname(build_file), f))) spec = targets[target] # Derive which gyp configuration to use based on MOZ_DEBUG. c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release' if c not in spec['configurations']: raise RuntimeError('Missing %s gyp configuration for target %s ' 'in %s' % (c, target_name, build_file)) target_conf = spec['configurations'][c] if 'actions' in spec: handle_actions(spec['actions'], context, action_overrides) if 'copies' in spec: handle_copies(spec['copies'], context) use_libs = [] libs = [] def add_deps(s): for t in s.get('dependencies', []) + s.get('dependencies_original', []): ty = targets[t]['type'] if ty in ('static_library', 'shared_library'): use_libs.append(targets[t]['target_name']) # Manually expand out transitive dependencies-- # gyp won't do this for static libs or none targets. if ty in ('static_library', 'none'): add_deps(targets[t]) libs.extend(spec.get('libraries', [])) #XXX: this sucks, but webrtc breaks with this right now because # it builds a library called 'gtest' and we just get lucky # that it isn't in USE_LIBS by that name anywhere. if no_chromium: add_deps(spec) os_libs = [] for l in libs: if l.startswith('-'): os_libs.append(l) elif l.endswith('.lib'): os_libs.append(l[:-4]) elif l: # For library names passed in from moz.build. use_libs.append(os.path.basename(l)) if spec['type'] == 'none': if not ('actions' in spec or 'copies' in spec): continue elif spec['type'] in ('static_library', 'shared_library', 'executable'): # Remove leading 'lib' from the target_name if any, and use as # library name. name = spec['target_name'] if spec['type'] in ('static_library', 'shared_library'): if name.startswith('lib'): name = name[3:] # The context expects an unicode string. context['LIBRARY_NAME'] = name.decode('utf-8') else: context['PROGRAM'] = name.decode('utf-8') if spec['type'] == 'shared_library': context['FORCE_SHARED_LIB'] = True elif spec['type'] == 'static_library' and spec.get( 'variables', {}).get('no_expand_libs', '0') == '1': # PSM links a NSS static library, but our folded libnss # doesn't actually export everything that all of the # objects within would need, so that one library # should be built as a real static library. context['NO_EXPAND_LIBS'] = True if use_libs: context['USE_LIBS'] = sorted(use_libs, key=lambda s: s.lower()) if os_libs: context['OS_LIBS'] = os_libs # gyp files contain headers and asm sources in sources lists. sources = [] unified_sources = [] extensions = set() use_defines_in_asflags = False for f in spec.get('sources', []): ext = mozpath.splitext(f)[-1] extensions.add(ext) if f.startswith('$INTERMEDIATE_DIR/'): s = ObjDirPath(context, f.replace('$INTERMEDIATE_DIR/', '!')) else: s = SourcePath(context, f) if ext == '.h': continue if ext == '.def': context['SYMBOLS_FILE'] = s elif ext != '.S' and not no_unified and s not in non_unified_sources: unified_sources.append(s) else: sources.append(s) # The Mozilla build system doesn't use DEFINES for building # ASFILES. if ext == '.s': use_defines_in_asflags = True # The context expects alphabetical order when adding sources context['SOURCES'] = alphabetical_sorted(sources) context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources) defines = target_conf.get('defines', []) if is_msvc and no_chromium: msvs_settings = gyp.msvs_emulation.MsvsSettings(spec, {}) defines.extend(msvs_settings.GetComputedDefines(c)) for define in defines: if '=' in define: name, value = define.split('=', 1) context['DEFINES'][name] = value else: context['DEFINES'][define] = True product_dir_dist = '$PRODUCT_DIR/dist/' for include in target_conf.get('include_dirs', []): if include.startswith(product_dir_dist): # special-case includes of <(PRODUCT_DIR)/dist/ to match # handle_copies above. This is used for NSS' exports. include = '!/dist/include/' + include[len(product_dir_dist ):] elif include.startswith(config.topobjdir): # NSPR_INCLUDE_DIR gets passed into the NSS build this way. include = '!/' + mozpath.relpath(include, config.topobjdir) else: # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do. # # NB: gyp files sometimes have actual absolute paths (e.g. # /usr/include32) and sometimes paths that moz.build considers # absolute, i.e. starting from topsrcdir. There's no good way # to tell them apart here, and the actual absolute paths are # likely bogus. In any event, actual absolute paths will be # filtered out by trying to find them in topsrcdir. if include.startswith('/'): resolved = mozpath.abspath( mozpath.join(config.topsrcdir, include[1:])) else: resolved = mozpath.abspath( mozpath.join(mozpath.dirname(build_file), include)) if not os.path.exists(resolved): continue context['LOCAL_INCLUDES'] += [include] context['ASFLAGS'] = target_conf.get('asflags_mozilla', []) if use_defines_in_asflags and defines: context['ASFLAGS'] += ['-D' + d for d in defines] flags = target_conf.get('cflags_mozilla', []) if flags: suffix_map = { '.c': 'CFLAGS', '.cpp': 'CXXFLAGS', '.cc': 'CXXFLAGS', '.m': 'CMFLAGS', '.mm': 'CMMFLAGS', } variables = (suffix_map[e] for e in extensions if e in suffix_map) for var in variables: for f in flags: # We may be getting make variable references out of the # gyp data, and we don't want those in emitted data, so # substitute them with their actual value. f = expand_variables(f, config.substs).split() if not f: continue # the result may be a string or a list. if isinstance(f, types.StringTypes): context[var].append(f) else: context[var].extend(f) else: # Ignore other types because we don't have # anything using them, and we're not testing them. They can be # added when that becomes necessary. raise NotImplementedError('Unsupported gyp target type: %s' % spec['type']) if not no_chromium: # Add some features to all contexts. Put here in case LOCAL_INCLUDES # order matters. context['LOCAL_INCLUDES'] += [ '!/ipc/ipdl/_ipdlheaders', '/ipc/chromium/src', '/ipc/glue', ] # These get set via VC project file settings for normal GYP builds. if is_win: context['DEFINES']['UNICODE'] = True context['DEFINES']['_UNICODE'] = True context['DISABLE_STL_WRAPPING'] = True yield context