Example #1
0
def grab_wheel(src, dst, source_sha, build):
  # late import lets us grab the virtualenv pip
  from pip.wheel import Wheel  # pylint: disable=E0611

  items = os.listdir(src)
  assert len(items) == 1, 'Wrong number of files in wheel directory: %r' % items

  wheelfile = items[0]
  wheel_info = Wheel.wheel_file_re.match(wheelfile)

  assert wheel_info is not None, 'Not a wheel file? %r' % wheelfile

  plat_tag = ''
  if not wheelfile.endswith('none-any.whl'):
    plat_tag = platform_tag()

  shutil.copyfile(
      os.path.join(src, wheelfile),
      os.path.join(dst, '{}-{}_{}{}{}'.format(
        wheel_info.group('namever'),
        build,
        source_sha,
        plat_tag,
        wheel_info.group(4),
      )))
Example #2
0
def grab_wheel(src, dst, source_sha, build):
    # late import lets us grab the virtualenv pip
    from pip.wheel import Wheel  # pylint: disable=E0611

    items = os.listdir(src)
    assert len(
        items) == 1, 'Wrong number of files in wheel directory: %r' % items

    wheelfile = items[0]
    wheel_info = Wheel.wheel_file_re.match(wheelfile)

    assert wheel_info is not None, 'Not a wheel file? %r' % wheelfile

    plat_tag = ''
    if not wheelfile.endswith('none-any.whl'):
        plat_tag = platform_tag()

    shutil.copyfile(
        os.path.join(src, wheelfile),
        os.path.join(
            dst, '{}-{}_{}{}{}'.format(
                wheel_info.group('namever'),
                build,
                source_sha,
                plat_tag,
                wheel_info.group(4),
            )))
Example #3
0
def main(args):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--deps_file',
        action='append',
        help='Path to deps.pyl file (default: bootstrap/deps.pyl)')
    parser.add_argument(
        'to_build',
        nargs='*',
        help='Names of packages to build. Defaults to all packages.')
    opts = parser.parse_args(args)

    if 'Ubuntu' in platform_tag() and ROOT.startswith('/usr/local/'):
        print >> sys.stderr, "\n".join([
            "Due to a bug in Ubuntu's python distribution, build_deps.py does not",
            "work when run from under a path beginning with /usr/local/. Please ",
            "clone to a different path, and try again.", "",
            "Bug: https://github.com/pypa/virtualenv/issues/118"
        ])
        return 1

    deps_files = opts.deps_file or [os.path.join(ROOT, 'deps.pyl')]
    to_build = set(opts.to_build)

    build_env = os.path.join(ROOT, 'BUILD_ENV')

    print 'Parsing deps.pyl'
    deps = merge_deps(deps_files)
    bootstrap.activate_env(build_env, {'wheel': deps.pop('wheel')})

    print 'Finding missing deps'
    missing_deps = {}
    for name, entry in deps.iteritems():
        if to_build and name not in to_build:
            continue
        try:
            bootstrap.get_links({name: entry})
        except bootstrap.NoWheelException:
            missing_deps[name] = entry

    if not missing_deps:
        print 'Nothing to process'
        return

    print 'Processing deps:'
    print_deps(missing_deps)

    for name, options in missing_deps.iteritems():
        clear_wheelhouse()
        # TODO(iannucci):  skip entries which already exist in gs
        if 'repo' in options and 'rev' in options:
            process_git(name, options['rev'], options['build'],
                        options.get('build_options', ()), options['repo'])
        elif 'gs' in options:
            process_gs(name, options['gs'], options['build'],
                       options.get('build_options', ()))
        else:
            raise Exception('Invalid options %r for %r' % (options, name))
        push_wheelhouse()
Example #4
0
def main(args):
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--deps_file', action='append',
      help='Path to deps.pyl file (default: bootstrap/deps.pyl)')
  parser.add_argument(
      'to_build', nargs='*',
      help='Names of packages to build. Defaults to all packages.')
  opts = parser.parse_args(args)

  if 'Ubuntu' in platform_tag() and ROOT.startswith('/usr/local/'):
    print >> sys.stderr, "\n".join([
      "Due to a bug in Ubuntu's python distribution, build_deps.py does not",
      "work when run from under a path beginning with /usr/local/. Please ",
      "clone to a different path, and try again.",
      "",
      "Bug: https://github.com/pypa/virtualenv/issues/118"
    ])
    return 1

  deps_files = opts.deps_file or [os.path.join(ROOT, 'deps.pyl')]
  to_build = set(opts.to_build)

  build_env = os.path.join(ROOT, 'BUILD_ENV')

  print 'Parsing deps.pyl'
  deps = merge_deps(deps_files)
  bootstrap.activate_env(build_env, {'wheel': deps.pop('wheel')})

  print 'Finding missing deps'
  missing_deps = {}
  for name, entry in deps.iteritems():
    if to_build and name not in to_build:
      continue
    try:
      bootstrap.get_links({name: entry})
    except bootstrap.NoWheelException:
      missing_deps[name] = entry

  if not missing_deps:
    print 'Nothing to process'
    return

  print 'Processing deps:'
  print_deps(missing_deps)

  for name, options in missing_deps.iteritems():
    clear_wheelhouse()
    # TODO(iannucci):  skip entries which already exist in gs
    if 'repo' in options and 'rev' in options:
      process_git(name, options['rev'], options['build'],
                  options.get('build_options', ()),
                  options['repo'])
    elif 'gs' in options:
      process_gs(name, options['gs'], options['build'],
                 options.get('build_options', ()))
    else:
      raise Exception('Invalid options %r for %r' % (options, name))
    push_wheelhouse()
Example #5
0
def get_links(deps):
    import pip.wheel  # pylint: disable=E0611
    plat_tag = platform_tag()  # this is something like '_Ubuntu_14.04' or ''

    links = []

    for name, dep in deps.iteritems():
        version, source_sha = dep['version'], sha_for(dep)
        prefix = '{}-{}-{}_{}'.format(name, version, dep['build'], source_sha)

        generic_wheels = []
        platform_wheels = []
        local_wheels = []

        for entry in ls('wheels/' + prefix):
            fname = entry['name'].split('/')[-1]
            wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname)
            if not wheel_info:
                LOGGER.warning('Skipping invalid wheel: %r', fname)
                continue

            # This check skips all obviously unsupported wheels (like Linux wheels on
            # Windows).
            if not pip.wheel.Wheel(fname).supported():
                continue

            if entry['local']:
                # A locally built wheel?
                local_wheels.append(entry)
            elif plat_tag and fname.startswith(prefix + plat_tag):
                # A wheel targeting our very specific platform (if any)? This is hit on
                # different versions of Ubuntu for example.
                platform_wheels.append(entry)
            else:
                # Some more generic wheel (e.g. 'linux1many' or source wheel).
                generic_wheels.append(entry)

        # Prefer local wheels if have them, then per-platform, then generic.
        wheel_set = local_wheels or platform_wheels or generic_wheels
        if not wheel_set:
            raise NoWheelException(name, version, dep['build'], source_sha)

        if len(wheel_set) != 1:
            LOGGER.warning('Letting pip choose a wheel for "%s":', name)
            for entry in wheel_set:
                LOGGER.warning(' * %s', entry['name'])

        links.extend(entry['link'] for entry in wheel_set)

    return links
Example #6
0
def get_links(deps):
    import pip.wheel  # pylint: disable=E0611
    plat_tag = platform_tag()

    links = []

    for name, dep in deps.iteritems():
        version, source_sha = dep['version'], sha_for(dep)
        prefix = 'wheels/{}-{}-{}_{}'.format(name, version, dep['build'],
                                             source_sha)
        generic_link = None
        binary_link = None
        local_link = None

        for entry in ls(prefix):
            fname = entry['name'].split('/')[-1]
            md5hash = entry['md5Hash']
            wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname)
            if not wheel_info:
                LOGGER.warn('Skipping invalid wheel: %r', fname)
                continue

            if pip.wheel.Wheel(fname).supported():
                if entry['local']:
                    link = LOCAL_OBJECT_URL.format(entry['name'])
                    local_link = link
                    continue
                else:
                    link = OBJECT_URL.format(entry['name'], md5hash)
                if fname.endswith('none-any.whl'):
                    if generic_link:
                        LOGGER.error(
                            'Found more than one generic matching wheel for %r: %r',
                            prefix, dep)
                        continue
                    generic_link = link
                elif plat_tag in fname:
                    if binary_link:
                        LOGGER.error(
                            'Found more than one binary matching wheel for %r: %r',
                            prefix, dep)
                        continue
                    binary_link = link

        if not binary_link and not generic_link and not local_link:
            raise NoWheelException(name, version, dep['build'], source_sha)

        links.append(local_link or binary_link or generic_link)

    return links
Example #7
0
def get_links(deps):
  import pip.wheel  # pylint: disable=E0611
  plat_tag = platform_tag()

  links = []

  for name, dep in deps.iteritems():
    version, source_sha = dep['version'] , sha_for(dep)
    prefix = 'wheels/{}-{}-{}_{}'.format(name, version, dep['build'],
                                         source_sha)
    generic_link = None
    binary_link = None
    local_link = None

    for entry in ls(prefix):
      fname = entry['name'].split('/')[-1]
      md5hash = entry['md5Hash']
      wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname)
      if not wheel_info:
        LOGGER.warn('Skipping invalid wheel: %r', fname)
        continue

      if pip.wheel.Wheel(fname).supported():
        if entry['local']:
          link = LOCAL_OBJECT_URL.format(entry['name'])
          local_link = link
          continue
        else:
          link = OBJECT_URL.format(entry['name'], md5hash)
        if fname.endswith('none-any.whl'):
          if generic_link:
            LOGGER.error(
              'Found more than one generic matching wheel for %r: %r',
              prefix, dep)
            continue
          generic_link = link
        elif plat_tag in fname:
          if binary_link:
            LOGGER.error(
              'Found more than one binary matching wheel for %r: %r',
              prefix, dep)
            continue
          binary_link = link

    if not binary_link and not generic_link and not local_link:
      raise NoWheelException(name, version, dep['build'], source_sha)

    links.append(local_link or binary_link or generic_link)

  return links