예제 #1
0
def main(args):
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument('revision', metavar='REVISION',
                      help='webports revision to to scan for.')
  parser.add_argument('-v', '--verbose', action='store_true',
                      help='Output extra information.')
  parser.add_argument('-p', '--parallel', action='store_true',
                      help='Download packages in parallel.')
  parser.add_argument('-l', '--cache-listing', action='store_true',
                      help='Cached output of gsutil -L (for testing).')
  parser.add_argument('--skip-md5', action='store_true',
                      help='Assume on-disk files are up-to-date (for testing).')
  args = parser.parse_args(args)
  if args.verbose:
    webports.set_verbose(True)

  sdk_version = webports.util.get_sdk_version()
  log('Scanning packages built for pepper_%s at revsion %s' %
      (sdk_version, args.revision))
  base_path = '%s/builds/pepper_%s/%s/publish' % (webports.GS_BUCKET,
                                                  sdk_version, args.revision)
  gs_base_url = 'gs://' + base_path
  cmd = find_gsutil() + ['ls', gs_base_url]
  log_verbose('Running: %s' % str(cmd))
  try:
    all_published = subprocess.check_output(cmd)
  except subprocess.CalledProcessError as e:
    raise webports.Error("Command '%s' failed: %s" % (cmd, e))

  pkg_dir = re.findall(r'pkg_[\w-]+', all_published)
  for pkg in pkg_dir:
    listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib',
                                pkg + '_' + 'listing.txt')
    if args.cache_listing and os.path.exists(listing_file):
      log('Using pre-cached gs listing: %s' % listing_file)
      with open(listing_file) as f:
        listing = f.read()
    else:
      gs_url = gs_base_url + '/' + pkg + '/*'
      log('Searching for packages at: %s' % gs_url)
      cmd = find_gsutil() + ['stat', gs_url]
      log_verbose('Running: %s' % str(cmd))
      try:
        listing = subprocess.check_output(cmd)
      except subprocess.CalledProcessError as e:
        raise webports.Error("Command '%s' failed: %s" % (cmd, e))
      if args.cache_listing:
        with open(listing_file, 'w') as f:
          f.write(listing)
    all_files = parse_gs_util_output(listing)
    log('Found %d packages [%s] for %s' %
        (len(all_files), format_size(sum(f.size for f in all_files)), pkg))
    download_files(pkg, all_files, not args.skip_md5, args.parallel)
  log('Done')
  return 0
예제 #2
0
def main(args):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('revision',
                        metavar='REVISION',
                        help='webports revision to to scan for.')
    parser.add_argument('-v',
                        '--verbose',
                        action='store_true',
                        help='Output extra information.')
    parser.add_argument('-p',
                        '--parallel',
                        action='store_true',
                        help='Download packages in parallel.')
    parser.add_argument('-l',
                        '--cache-listing',
                        action='store_true',
                        help='Cached output of gsutil -L (for testing).')
    parser.add_argument(
        '--skip-md5',
        action='store_true',
        help='Assume on-disk files are up-to-date (for testing).')
    args = parser.parse_args(args)
    if args.verbose:
        webports.set_verbose(True)

    sdk_version = webports.util.get_sdk_version()
    log('Scanning packages built for pepper_%s at revsion %s' %
        (sdk_version, args.revision))
    base_path = '%s/builds/pepper_%s/%s/publish' % (webports.GS_BUCKET,
                                                    sdk_version, args.revision)
    gs_base_url = 'gs://' + base_path
    cmd = find_gsutil() + ['ls', gs_base_url]
    log_verbose('Running: %s' % str(cmd))
    try:
        all_published = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as e:
        raise webports.Error("Command '%s' failed: %s" % (cmd, e))

    pkg_dir = re.findall(r'pkg_[\w-]+', all_published)
    for pkg in pkg_dir:
        listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib',
                                    pkg + '_' + 'listing.txt')
        if args.cache_listing and os.path.exists(listing_file):
            log('Using pre-cached gs listing: %s' % listing_file)
            with open(listing_file) as f:
                listing = f.read()
        else:
            gs_url = gs_base_url + '/' + pkg + '/*'
            log('Searching for packages at: %s' % gs_url)
            cmd = find_gsutil() + ['stat', gs_url]
            log_verbose('Running: %s' % str(cmd))
            try:
                listing = subprocess.check_output(cmd)
            except subprocess.CalledProcessError as e:
                raise webports.Error("Command '%s' failed: %s" % (cmd, e))
            if args.cache_listing:
                with open(listing_file, 'w') as f:
                    f.write(listing)
        all_files = parse_gs_util_output(listing)
        log('Found %d packages [%s] for %s' %
            (len(all_files), format_size(sum(f.size for f in all_files)), pkg))
        download_files(pkg, all_files, not args.skip_md5, args.parallel)
    log('Done')
    return 0
예제 #3
0
def download_files(pkg_dir, files, check_hashes=True, parallel=False):
  """Download one of more files to the local disk.

  Args:
    files: List of FileInfo objects to download.
    check_hashes: When False assume local files have the correct
    hash otherwise always check the hashes match the onces in the
    FileInfo ojects.

  Returns:
    List of (filename, url) tuples.
  """
  files_to_download = []
  filenames = []
  download_dir = os.path.join(webports.package_index.PREBUILT_ROOT, 'pkg',
                              pkg_dir)
  if not os.path.exists(download_dir):
    os.makedirs(download_dir)

  for file_info in files:
    basename = os.path.basename(file_info.url)
    # TODO(bradnelson): The pnacl package is >2GB, which confuses pkg.
    #                   Figure out why and re-enable this.
    if basename == 'pnacl-0.0.1.tbz':
      continue
    file_info.name = os.path.join(download_dir, basename)
    file_info.rel_name = file_info.name[len(webports.paths.NACLPORTS_ROOT) + 1:]
    filenames.append((file_info.name, file_info.url))
    if os.path.exists(file_info.name):
      if not check_hashes or check_hash(file_info.name, file_info.md5):
        log('Up-to-date: %s' % file_info.rel_name)
        continue
    files_to_download.append(file_info)

  def check(file_info):
    if check_hashes and not check_hash(file_info.name, file_info.md5):
      raise webports.Error(
          'Checksum failed: %s\nExpected=%s\nActual=%s' %
          (file_info.rel_name, file_info.md5, get_hash(file_info.name)))

  if not files_to_download:
    log('All files up-to-date')
  else:
    total_size = sum(f.size for f in files_to_download)
    log('Need to download %d/%d files [%s]' %
        (len(files_to_download), len(files), format_size(total_size)))

    gsutil = find_gsutil()
    if parallel:
      remaining_files = files_to_download
      num_files = 20
      while remaining_files:
        files = remaining_files[:num_files]
        remaining_files = remaining_files[num_files:]
        cmd = gsutil + ['-m', 'cp'] + [f.gsurl for f in files] + [download_dir]
        log_verbose(cmd)
        subprocess.check_call(cmd)
        for file_info in files:
          check(file_info)
    else:
      for file_info in files_to_download:
        webports.download_file(file_info.name, file_info.url)
        check(file_info)

  return filenames
예제 #4
0
def download_files(pkg_dir, files, check_hashes=True, parallel=False):
    """Download one of more files to the local disk.

  Args:
    files: List of FileInfo objects to download.
    check_hashes: When False assume local files have the correct
    hash otherwise always check the hashes match the onces in the
    FileInfo ojects.

  Returns:
    List of (filename, url) tuples.
  """
    files_to_download = []
    filenames = []
    download_dir = os.path.join(webports.package_index.PREBUILT_ROOT, 'pkg',
                                pkg_dir)
    if not os.path.exists(download_dir):
        os.makedirs(download_dir)

    for file_info in files:
        basename = os.path.basename(file_info.url)
        # TODO(bradnelson): The pnacl package is >2GB, which confuses pkg.
        #                   Figure out why and re-enable this.
        if basename == 'pnacl-0.0.1.tbz':
            continue
        file_info.name = os.path.join(download_dir, basename)
        file_info.rel_name = file_info.name[len(webports.paths.NACLPORTS_ROOT
                                                ) + 1:]
        filenames.append((file_info.name, file_info.url))
        if os.path.exists(file_info.name):
            if not check_hashes or check_hash(file_info.name, file_info.md5):
                log('Up-to-date: %s' % file_info.rel_name)
                continue
        files_to_download.append(file_info)

    def check(file_info):
        if check_hashes and not check_hash(file_info.name, file_info.md5):
            raise webports.Error(
                'Checksum failed: %s\nExpected=%s\nActual=%s' %
                (file_info.rel_name, file_info.md5, get_hash(file_info.name)))

    if not files_to_download:
        log('All files up-to-date')
    else:
        total_size = sum(f.size for f in files_to_download)
        log('Need to download %d/%d files [%s]' %
            (len(files_to_download), len(files), format_size(total_size)))

        gsutil = find_gsutil()
        if parallel:
            remaining_files = files_to_download
            num_files = 20
            while remaining_files:
                files = remaining_files[:num_files]
                remaining_files = remaining_files[num_files:]
                cmd = gsutil + ['-m', 'cp'] + [f.gsurl for f in files
                                               ] + [download_dir]
                log_verbose(cmd)
                subprocess.check_call(cmd)
                for file_info in files:
                    check(file_info)
        else:
            for file_info in files_to_download:
                webports.download_file(file_info.name, file_info.url)
                check(file_info)

    return filenames