Esempio n. 1
0
def build_pex(args, options, resolver_option_builder):
  with TRACER.timed('Resolving interpreter', V=2):
    interpreter = interpreter_from_options(options)

  if interpreter is None:
    die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)

  pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter)

  pex_info = pex_builder.info
  pex_info.zip_safe = options.zip_safe
  pex_info.always_write_cache = options.always_write_cache
  pex_info.ignore_errors = options.ignore_errors
  pex_info.inherit_path = options.inherit_path

  resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]

  for requirements_txt in options.requirement_files:
    resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))

  # pip states the constraints format is identical tor requirements
  # https://pip.pypa.io/en/stable/user_guide/#constraints-files
  for constraints_txt in options.constraint_files:
    constraints = []
    for r in requirements_from_file(constraints_txt, resolver_option_builder):
      r.is_constraint = True
      constraints.append(r)
    resolvables.extend(constraints)

  resolver_kwargs = dict(interpreter=interpreter, platform=options.platform)

  if options.cache_dir:
    resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs)
  else:
    resolver = Resolver(**resolver_kwargs)

  with TRACER.timed('Resolving distributions'):
    try:
      resolveds = resolver.resolve(resolvables)
    except Unsatisfiable as e:
      die(e)

  for dist in resolveds:
    log('  %s' % dist, v=options.verbosity)
    pex_builder.add_distribution(dist)
    pex_builder.add_requirement(dist.as_requirement())

  if options.entry_point and options.script:
    die('Must specify at most one entry point or script.', INVALID_OPTIONS)

  if options.entry_point:
    pex_builder.set_entry_point(options.entry_point)
  elif options.script:
    pex_builder.set_script(options.script)

  if options.python_shebang:
    pex_builder.set_shebang(options.python_shebang)

  return pex_builder
Esempio n. 2
0
File: pex.py Progetto: windie/heron
def build_pex(args, options, resolver_option_builder, interpreter=None):
    if interpreter is None:
        with TRACER.timed('Resolving interpreter', V=2):
            interpreter = interpreter_from_options(options)

    if interpreter is None:
        die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)

    pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter)

    pex_info = pex_builder.info
    pex_info.zip_safe = options.zip_safe
    pex_info.always_write_cache = options.always_write_cache
    pex_info.ignore_errors = options.ignore_errors
    pex_info.inherit_path = options.inherit_path

    resolvables = [
        Resolvable.get(arg, resolver_option_builder) for arg in args
    ]

    for requirements_txt in options.requirement_files:
        resolvables.extend(
            requirements_from_file(requirements_txt, resolver_option_builder))

    resolver_kwargs = dict(interpreter=interpreter, platform=options.platform)

    if options.cache_dir:
        resolver = CachingResolver(options.cache_dir, options.cache_ttl,
                                   **resolver_kwargs)
    else:
        resolver = Resolver(**resolver_kwargs)

    with TRACER.timed('Resolving distributions'):
        try:
            resolveds = resolver.resolve(resolvables)
        except Unsatisfiable as e:
            die(e)

    for dist in resolveds:
        log('  %s' % dist, v=options.verbosity)
        pex_builder.add_distribution(dist)
        pex_builder.add_requirement(dist.as_requirement())

    if options.entry_point and options.script:
        die('Must specify at most one entry point or script.', INVALID_OPTIONS)

    if options.entry_point:
        pex_builder.set_entry_point(options.entry_point)
    elif options.script:
        pex_builder.set_script(options.script)

    if options.python_shebang:
        pex_builder.set_shebang(options.python_shebang)

    return pex_builder
Esempio n. 3
0
File: pex.py Progetto: twitter/heron
def build_pex(args, options, resolver_option_builder, interpreter=None):
    if interpreter is None:
        with TRACER.timed("Resolving interpreter", V=2):
            interpreter = interpreter_from_options(options)

    if interpreter is None:
        die("Could not find compatible interpreter", CANNOT_SETUP_INTERPRETER)

    pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter)

    pex_info = pex_builder.info
    pex_info.zip_safe = options.zip_safe
    pex_info.always_write_cache = options.always_write_cache
    pex_info.ignore_errors = options.ignore_errors
    pex_info.inherit_path = options.inherit_path

    resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args]

    for requirements_txt in options.requirement_files:
        resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))

    resolver_kwargs = dict(interpreter=interpreter, platform=options.platform)

    if options.cache_dir:
        resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs)
    else:
        resolver = Resolver(**resolver_kwargs)

    with TRACER.timed("Resolving distributions"):
        try:
            resolveds = resolver.resolve(resolvables)
        except Unsatisfiable as e:
            die(e)

    for dist in resolveds:
        log("  %s" % dist, v=options.verbosity)
        pex_builder.add_distribution(dist)
        pex_builder.add_requirement(dist.as_requirement())

    if options.entry_point and options.script:
        die("Must specify at most one entry point or script.", INVALID_OPTIONS)

    if options.entry_point:
        pex_builder.set_entry_point(options.entry_point)
    elif options.script:
        pex_builder.set_script(options.script)

    if options.python_shebang:
        pex_builder.set_shebang(options.python_shebang)

    return pex_builder
Esempio n. 4
0
def build_pex(args):
    with TRACER.timed('Resolving interpreter', V=2):
        interpreter = _establish_interpreter(args)

    if interpreter is None:
        die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER)

    pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=_PREAMBLE)

    pex_info = pex_builder.info

    pex_info.zip_safe = False
    pex_info.always_write_cache = True
    pex_info.inherit_path = False

    resolver_option_builder = _establish_resolver_options(args)
    reqs = args.reqs
    resolvables = [Resolvable.get(req, resolver_option_builder) for req in reqs]

    for requirements_txt in args.requirement_files:
        resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder))

    resolver_kwargs = dict(interpreter=interpreter, platform=args.platform)
    _add_spex_deps(resolvables, pex_builder, resolver_option_builder=resolver_option_builder)

    if not args.disable_cache:
        resolver = CachingResolver(args.cache_dir, args.cache_ttl, **resolver_kwargs)
    else:
        resolver = Resolver(**resolver_kwargs)

    resolveds = []
    with TRACER.timed('Resolving distributions'):
        try:
            resolveds = resolver.resolve(resolvables)
        except Unsatisfiable as exception:
            die(exception)

    for dist in resolveds:
        log('  %s' % dist, verbose=args.verbosity)
        pex_builder.add_distribution(dist)
        pex_builder.add_requirement(dist.as_requirement())

    pex_builder.set_entry_point('spex:spex')

    if args.python_shebang:
        pex_builder.set_shebang(args.python_shebang)

    return pex_builder
Esempio n. 5
0
def build_site_packages():
    """Use PEX to resolve dependencies in a virtual environment,
    with some customizations to reduce the size of our build.

    https://www.pantsbuild.org/pex.html
    """
    # Remove flywheel_cli from cache
    # If you skip this step, it doesn't automatically update the python code
    if os.path.isdir(PEX_BUILD_CACHE_DIR):
        for name in os.listdir(PEX_BUILD_CACHE_DIR):
            if fnmatch.fnmatch(name, 'flywheel_cli*.whl'):
                path = os.path.join(PEX_BUILD_CACHE_DIR, name)
                print('Removing {} from cache...'.format(name))
                os.remove(path)

    # Read ignore list
    # See package-ignore.txt, largely we're removing test files and
    # Multi-megabyte dicoms from the dicom folder
    ignore_patterns = read_ignore_patterns()

    # Create resolver
    # Loosely based on: https://github.com/pantsbuild/pex/blob/982cb9a988949ffff3348b9bca98ae72a0bf8847/pex/bin/pex.py#L577
    resolver_option_builder = ResolverOptionsBuilder()
    resolvables = [
        Resolvable.get('flywheel-cli=={}'.format(PYTHON_CLI_VERSION),
                       resolver_option_builder)
    ]
    resolver = CachingResolver(PEX_BUILD_CACHE_DIR, None)

    # Effectively we resolve (possibly from cache) The source and all of the dependency packages
    # Then create the virtual environment, which contains those files
    print('Resolving distributions')
    resolved = resolver.resolve(resolvables)

    print('Building package lists')
    builder = PEXBuilder()
    for dist in resolved:
        builder.add_distribution(dist)
        builder.add_requirement(dist.as_requirement())

    # After this point, builder.chroot contains a full list of the files
    print('Compiling package')
    builder.freeze(bytecode_compile=False)

    site_packages_path = os.path.join(BUILD_DIR, 'site-packages.zip')

    # Create an uncompressed site-packages.zip and add all of the discovered files
    # (Except those that are filtered out)
    with open(site_packages_path, 'wb') as f:
        added_files = set()
        with zipfile.ZipFile(f, 'w') as zf:
            for filename in sorted(builder.chroot().files()):
                if is_ignored_file(ignore_patterns, filename):
                    continue

                if not filename.startswith('.deps'):
                    continue

                # Determine new path
                src_path = os.path.join(builder.chroot().chroot, filename)
                dst_path = '/'.join(filename.split('/')[2:])

                # Optionally, compile the file
                _, ext = os.path.splitext(src_path)
                if ext == '.py':
                    cfile_path = src_path + 'c'
                    dst_path += 'c'

                    print('Compiling: {}'.format(dst_path))
                    py_compile.compile(src_path,
                                       cfile=cfile_path,
                                       dfile=dst_path,
                                       optimize=1)
                    src_path = cfile_path

                if not dst_path in added_files:
                    zf.write(src_path, dst_path)
                    added_files.add(dst_path)

    return site_packages_path